{"id":"https://openalex.org/W2144139817","doi":"https://doi.org/10.1109/tasl.2008.2011509","title":"Cross-Modality Semantic Integration With Hypothesis Rescoring for Robust Interpretation of Multimodal User Interactions","display_name":"Cross-Modality Semantic Integration With Hypothesis Rescoring for Robust Interpretation of Multimodal User Interactions","publication_year":2009,"publication_date":"2009-02-13","ids":{"openalex":"https://openalex.org/W2144139817","doi":"https://doi.org/10.1109/tasl.2008.2011509","mag":"2144139817"},"language":"en","primary_location":{"id":"doi:10.1109/tasl.2008.2011509","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tasl.2008.2011509","pdf_url":null,"source":{"id":"https://openalex.org/S199497470","display_name":"IEEE Transactions on Audio Speech and Language Processing","issn_l":"1558-7916","issn":["1558-7916","1558-7924"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Audio, Speech, and Language Processing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5087604485","display_name":"Pui-Yu Hui","orcid":null},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Pui-Yu Hui","raw_affiliation_strings":["Human-Computer Communications Laboratory, Department of Systems Engineering and Engineering Management, Chinese University of Hong Kong, Sha Tin, Hong Kong, China","Dept. of Syst. Eng. & Eng. Manage., Chinese Univ. of Hong Kong, Hong Kong#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Human-Computer Communications Laboratory, Department of Systems Engineering and Engineering Management, Chinese University of Hong Kong, Sha Tin, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]},{"raw_affiliation_string":"Dept. of Syst. Eng. & Eng. Manage., Chinese Univ. of Hong Kong, Hong Kong#TAB#","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5102225156","display_name":"Helen Meng","orcid":"https://orcid.org/0000-0002-4427-3532"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Helen M. Meng","raw_affiliation_strings":["Dept. of Syst. Eng. & Eng. Manage., Chinese Univ. of Hong Kong, Hong Kong#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Dept. of Syst. Eng. & Eng. Manage., Chinese Univ. of Hong Kong, Hong Kong#TAB#","institution_ids":["https://openalex.org/I177725633"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I177725633"],"apc_list":null,"apc_paid":null,"fwci":0.5221,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.79638828,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":94},"biblio":{"volume":"17","issue":"3","first_page":"486","last_page":"500"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/gesture","display_name":"Gesture","score":0.6594250798225403},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6168789863586426},{"id":"https://openalex.org/keywords/event","display_name":"Event (particle physics)","score":0.5392730236053467},{"id":"https://openalex.org/keywords/interpretation","display_name":"Interpretation (philosophy)","score":0.5014986991882324},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4744842052459717},{"id":"https://openalex.org/keywords/semantics","display_name":"Semantics (computer science)","score":0.4347579777240753},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4300720989704132},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3504061698913574},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.18962159752845764}],"concepts":[{"id":"https://openalex.org/C207347870","wikidata":"https://www.wikidata.org/wiki/Q371174","display_name":"Gesture","level":2,"score":0.6594250798225403},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6168789863586426},{"id":"https://openalex.org/C2779662365","wikidata":"https://www.wikidata.org/wiki/Q5416694","display_name":"Event (particle physics)","level":2,"score":0.5392730236053467},{"id":"https://openalex.org/C527412718","wikidata":"https://www.wikidata.org/wiki/Q855395","display_name":"Interpretation (philosophy)","level":2,"score":0.5014986991882324},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4744842052459717},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.4347579777240753},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4300720989704132},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3504061698913574},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.18962159752845764},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tasl.2008.2011509","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tasl.2008.2011509","pdf_url":null,"source":{"id":"https://openalex.org/S199497470","display_name":"IEEE Transactions on Audio Speech and Language Processing","issn_l":"1558-7916","issn":["1558-7916","1558-7924"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Audio, Speech, and Language Processing","raw_type":"journal-article"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.148.9689","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.148.9689","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.se.cuhk.edu.hk/hccl/publications/pub/tasl-hui-2011509.pdf","raw_type":"text"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320322724","display_name":"Ministry of Education, India","ror":"https://ror.org/048xjjh50"},{"id":"https://openalex.org/F4320322942","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":28,"referenced_works":["https://openalex.org/W10704533","https://openalex.org/W32727853","https://openalex.org/W108550785","https://openalex.org/W158869525","https://openalex.org/W1488550131","https://openalex.org/W1549285799","https://openalex.org/W1551281040","https://openalex.org/W1968932197","https://openalex.org/W1998632015","https://openalex.org/W1999090433","https://openalex.org/W2006969979","https://openalex.org/W2009803366","https://openalex.org/W2013738217","https://openalex.org/W2040646018","https://openalex.org/W2083325968","https://openalex.org/W2095245616","https://openalex.org/W2104851574","https://openalex.org/W2116190584","https://openalex.org/W2151194828","https://openalex.org/W2166858357","https://openalex.org/W2196854674","https://openalex.org/W2279606232","https://openalex.org/W2791309451","https://openalex.org/W3144650235","https://openalex.org/W6601351737","https://openalex.org/W6604322312","https://openalex.org/W6606494744","https://openalex.org/W6652311901"],"related_works":["https://openalex.org/W2066003895","https://openalex.org/W2537963312","https://openalex.org/W2537762514","https://openalex.org/W2349788282","https://openalex.org/W577271088","https://openalex.org/W2120801881","https://openalex.org/W1974473538","https://openalex.org/W2020010749","https://openalex.org/W2164899521","https://openalex.org/W1982853263"],"abstract_inverted_index":{"We":[0,64],"develop":[1],"a":[2,58,84,132],"framework":[3,47],"pertaining":[4],"to":[5,122,147,187,200,255],"automatic":[6,178],"semantic":[7,108],"interpretation":[8],"of":[9,39,61,70,100,110,117,138,142,151,167,210,229,240],"multimodal":[10,85,125,133],"user":[11],"interactions":[12],"using":[13],"speech":[14,35,75,153,179],"and":[15,78,154,170,180,189,194,244,257,262],"pen":[16,43,80,155,181,242],"gestures.":[17],"The":[18,45,113,221],"two":[19],"input":[20,29,55,76,81,102],"modalities":[21,156],"abstract":[22],"the":[23,42,68,91,97,101,107,123,136,143,152,168,184,192,225,249,253,260],"user's":[24],"intended":[25],"message":[26],"differently":[27],"into":[28],"events,":[30],"e.g.,":[31],"key":[32],"terms/phrases":[33],"in":[34,41,83,135],"or":[36],"different":[37],"types":[38],"gestures":[40,243],"modality.":[44,220],"proposed":[46],"begins":[48],"by":[49,90],"generating":[50],"partial":[51],"interpretations":[52],"for":[53,164,191,259],"each":[54,219],"event":[56,77,82],"as":[57,104,106,234,236],"ranked":[59],"list":[60],"hypothesized":[62],"semantics.":[63],"devise":[65],"across-modalitysemanticintegrationprocedureto":[66],"align":[67],"pair":[69],"hypothesis":[71],"lists":[72],"between":[73,238],"every":[74,79],"expression.":[86,126],"This":[87],"is":[88,119],"achieved":[89],"Viterbi":[92],"alignment":[93,114],"algorithm":[94],"that":[95,158],"enforces":[96],"temporal":[98],"ordering":[99],"events":[103],"well":[105,235],"compatibility":[109],"aligned":[111],"events.":[112],"enables":[115],"generation":[116],"aunimodal,verbalizedparaphrasethat":[118],"semantically":[120],"equivalent":[121],"original":[124],"Our":[127],"experiments":[128],"are":[129,162],"based":[130],"on":[131,248],"corpus":[134],"domain":[137],"city":[139],"navigation.":[140],"Application":[141,251],"cross-modality":[144,211],"integration":[145,212,226],"procedure":[146],"near-perfect":[148],"(manual)":[149],"transcripts":[150],"show":[157],"correct":[159],"unimodal":[160],"paraphrases":[161],"generated":[163],"over":[165],"97%":[166],"training":[169,193,261],"test":[171,195,263],"sets.":[172],"However,":[173],"if":[174],"we":[175,204],"replace":[176],"with":[177],"recognition":[182,216],"transcripts,":[183],"performance":[185,254],"drops":[186],"53.7%":[188],"54.8%":[190],"sets,":[196,264],"respectively.":[197,265],"In":[198],"order":[199],"address":[201],"this":[202],"issue,":[203],"devised":[205],"thehypothesisrescoringprocedurethat":[206],"evaluates":[207],"all":[208],"candidates":[209],"derived":[213],"from":[214,218],"multiple":[215],"hypotheses":[217],"rescoring":[222],"function":[223],"incorporates":[224],"score,N-best":[227],"purity":[228],"recognized":[230,241],"spoken":[231],"locative":[232],"expressions,":[233],"distances":[237],"coordinates":[239],"their":[245],"interpreted":[246],"icons":[247],"map.":[250],"ofcross-modalityhypothesisrescoringimproved":[252],"67.5%":[256],"69.9%":[258]},"counts_by_year":[{"year":2014,"cited_by_count":1}],"updated_date":"2026-07-29T09:40:50.615796","created_date":"2025-10-10T00:00:00"}