{"id":"https://openalex.org/W2144139817","doi":"https://doi.org/10.1109/tasl.2008.2011509","title":"Cross-Modality Semantic Integration With Hypothesis Rescoring for Robust Interpretation of Multimodal User Interactions","display_name":"Cross-Modality Semantic Integration With Hypothesis Rescoring for Robust Interpretation of Multimodal User Interactions","publication_year":2009,"publication_date":"2009-02-13","ids":{"openalex":"https://openalex.org/W2144139817","doi":"https://doi.org/10.1109/tasl.2008.2011509","mag":"2144139817"},"language":"en","primary_location":{"id":"doi:10.1109/tasl.2008.2011509","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tasl.2008.2011509","pdf_url":null,"source":{"id":"https://openalex.org/S199497470","display_name":"IEEE Transactions on Audio Speech and Language Processing","issn_l":"1558-7916","issn":["1558-7916","1558-7924"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Audio, Speech, and Language Processing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5087604485","display_name":"Pui-Yu Hui","orcid":null},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"CN","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Pui-Yu Hui","raw_affiliation_strings":["Human-Computer Communications Laboratory, Department of Systems Engineering and Engineering Management, Chinese University of Hong Kong, Sha Tin, Hong Kong, China","Dept. of Syst. Eng. & Eng. Manage., Chinese Univ. of Hong Kong, Hong Kong#TAB#"],"affiliations":[{"raw_affiliation_string":"Human-Computer Communications Laboratory, Department of Systems Engineering and Engineering Management, Chinese University of Hong Kong, Sha Tin, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]},{"raw_affiliation_string":"Dept. of Syst. Eng. & Eng. Manage., Chinese Univ. of Hong Kong, Hong Kong#TAB#","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5102225156","display_name":"Helen Meng","orcid":null},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"CN","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Helen M. Meng","raw_affiliation_strings":["Human-Computer Communications Laboratory, Department of Systems Engineering and Engineering Management, Chinese University of Hong Kong, Sha Tin, Hong Kong, China","Dept. of Syst. Eng. & Eng. Manage., Chinese Univ. of Hong Kong, Hong Kong#TAB#"],"affiliations":[{"raw_affiliation_string":"Human-Computer Communications Laboratory, Department of Systems Engineering and Engineering Management, Chinese University of Hong Kong, Sha Tin, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]},{"raw_affiliation_string":"Dept. of Syst. Eng. & Eng. Manage., Chinese Univ. of Hong Kong, Hong Kong#TAB#","institution_ids":["https://openalex.org/I177725633"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5087604485"],"corresponding_institution_ids":["https://openalex.org/I177725633"],"apc_list":null,"apc_paid":null,"fwci":0.4516,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.75192861,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":94},"biblio":{"volume":"17","issue":"3","first_page":"486","last_page":"500"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/gesture","display_name":"Gesture","score":0.6594250798225403},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6168789863586426},{"id":"https://openalex.org/keywords/event","display_name":"Event (particle physics)","score":0.5392730236053467},{"id":"https://openalex.org/keywords/interpretation","display_name":"Interpretation (philosophy)","score":0.5014986991882324},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4744842052459717},{"id":"https://openalex.org/keywords/semantics","display_name":"Semantics (computer science)","score":0.4347579777240753},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4300720989704132},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3504061698913574},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.18962159752845764}],"concepts":[{"id":"https://openalex.org/C207347870","wikidata":"https://www.wikidata.org/wiki/Q371174","display_name":"Gesture","level":2,"score":0.6594250798225403},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6168789863586426},{"id":"https://openalex.org/C2779662365","wikidata":"https://www.wikidata.org/wiki/Q5416694","display_name":"Event (particle physics)","level":2,"score":0.5392730236053467},{"id":"https://openalex.org/C527412718","wikidata":"https://www.wikidata.org/wiki/Q855395","display_name":"Interpretation (philosophy)","level":2,"score":0.5014986991882324},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4744842052459717},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.4347579777240753},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4300720989704132},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3504061698913574},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.18962159752845764},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tasl.2008.2011509","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tasl.2008.2011509","pdf_url":null,"source":{"id":"https://openalex.org/S199497470","display_name":"IEEE Transactions on Audio Speech and Language Processing","issn_l":"1558-7916","issn":["1558-7916","1558-7924"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Audio, Speech, and Language Processing","raw_type":"journal-article"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.148.9689","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.148.9689","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.se.cuhk.edu.hk/hccl/publications/pub/tasl-hui-2011509.pdf","raw_type":"text"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320322724","display_name":"Ministry of Education, India","ror":"https://ror.org/048xjjh50"},{"id":"https://openalex.org/F4320322942","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":28,"referenced_works":["https://openalex.org/W10704533","https://openalex.org/W32727853","https://openalex.org/W108550785","https://openalex.org/W158869525","https://openalex.org/W1488550131","https://openalex.org/W1549285799","https://openalex.org/W1551281040","https://openalex.org/W1968932197","https://openalex.org/W1998632015","https://openalex.org/W1999090433","https://openalex.org/W2006969979","https://openalex.org/W2009803366","https://openalex.org/W2013738217","https://openalex.org/W2040646018","https://openalex.org/W2083325968","https://openalex.org/W2095245616","https://openalex.org/W2104851574","https://openalex.org/W2116190584","https://openalex.org/W2151194828","https://openalex.org/W2166858357","https://openalex.org/W2196854674","https://openalex.org/W2279606232","https://openalex.org/W2791309451","https://openalex.org/W3144650235","https://openalex.org/W6601351737","https://openalex.org/W6604322312","https://openalex.org/W6606494744","https://openalex.org/W6652311901"],"related_works":["https://openalex.org/W2066003895","https://openalex.org/W2537963312","https://openalex.org/W2537762514","https://openalex.org/W2349788282","https://openalex.org/W577271088","https://openalex.org/W2120801881","https://openalex.org/W1974473538","https://openalex.org/W2020010749","https://openalex.org/W2164899521","https://openalex.org/W1982853263"],"abstract_inverted_index":{"We":[0,64],"develop":[1],"a":[2,58,66,97,131,156],"framework":[3,47],"pertaining":[4],"to":[5,79,146,171,211,224,303],"automatic":[6,202],"semantic":[7,121],"interpretation":[8],"of":[9,39,61,83,113,123,130,162,166,175,191,244,267,278,290],"multimodal":[10,98,149,157],"user":[11],"interactions":[12],"using":[13],"speech":[14,35,88,177,203],"and":[15,91,178,194,204,213,218,282,305,310],"pen":[16,43,93,179,205,280],"gestures.":[17],"The":[18,45,126,255],"two":[19],"input":[20,29,55,89,94,115],"modalities":[21,180],"abstract":[22],"the":[23,42,81,104,110,114,120,147,160,167,176,192,208,216,230,259,287,301,308],"user's":[24],"intended":[25],"message":[26],"differently":[27],"into":[28],"events,":[30],"e.g.,":[31],"key":[32],"terms/phrases":[33],"in":[34,41,96,159],"or":[36],"different":[37],"types":[38],"gestures":[40,281],"modality.":[44,254],"proposed":[46],"begins":[48],"by":[49,103],"generating":[50],"partial":[51],"interpretations":[52],"for":[53,188,215,307],"each":[54,253],"event":[56,90,95],"as":[57,117,119,272,274],"ranked":[59],"list":[60],"hypothesized":[62],"semantics.":[63],"devise":[65],"<i":[67,70,73,76,132,136,139,231,234,237,262,291,294,297],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[68,71,74,77,133,137,140,232,235,238,263,292,295,298],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">cross-modality</i>":[69,293],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">semantic</i>":[72],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">integration</i>":[75],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">procedure</i>":[78,239],"align":[80],"pair":[82],"hypothesis":[84],"lists":[85],"between":[86,276],"every":[87,92],"expression.":[99,150],"This":[100],"is":[101,143],"achieved":[102],"Viterbi":[105],"alignment":[106,127],"algorithm":[107],"that":[108,142,182,240],"enforces":[109],"temporal":[111],"ordering":[112],"events":[116],"well":[118,273],"compatibility":[122],"aligned":[124],"events.":[125],"enables":[128],"generation":[129],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">unimodal</i>":[134],",":[135],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">verbalized</i>":[138],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">paraphrase</i>":[141],"semantically":[144],"equivalent":[145],"original":[148],"Our":[151],"experiments":[152],"are":[153,186],"based":[154],"on":[155,286],"corpus":[158],"domain":[161],"city":[163],"navigation.":[164],"Application":[165,289],"cross-modality":[168,245],"integration":[169,246,260],"procedure":[170],"near-perfect":[172],"(manual)":[173],"transcripts":[174],"show":[181],"correct":[183],"unimodal":[184],"paraphrases":[185],"generated":[187],"over":[189],"97%":[190],"training":[193,217,309],"test":[195,219,311],"sets.":[196],"However,":[197],"if":[198],"we":[199,228],"replace":[200],"with":[201],"recognition":[206,250],"transcripts,":[207],"performance":[209,302],"drops":[210],"53.7%":[212],"54.8%":[214],"sets,":[220,312],"respectively.":[221,313],"In":[222],"order":[223],"address":[225],"this":[226],"issue,":[227],"devised":[229],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">hypothesis</i>":[233,296],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">rescoring</i>":[236,299],"evaluates":[241],"all":[242],"candidates":[243],"derived":[247],"from":[248,252],"multiple":[249],"hypotheses":[251],"rescoring":[256],"function":[257],"incorporates":[258],"score,":[261],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">N</i>":[264],"-best":[265],"purity":[266],"recognized":[268,279],"spoken":[269],"locative":[270],"expressions,":[271],"distances":[275],"coordinates":[277],"their":[283],"interpreted":[284],"icons":[285],"map.":[288],"improved":[300],"67.5%":[304],"69.9%":[306]},"counts_by_year":[{"year":2014,"cited_by_count":1}],"updated_date":"2026-04-05T17:49:38.594831","created_date":"2025-10-10T00:00:00"}