{"id":"https://openalex.org/W4390099706","doi":"https://doi.org/10.1109/robio58561.2023.10354726","title":"Speech-image based Multimodal AI Interaction for Scrub Nurse Assistance in the Operating Room","display_name":"Speech-image based Multimodal AI Interaction for Scrub Nurse Assistance in the Operating Room","publication_year":2023,"publication_date":"2023-12-04","ids":{"openalex":"https://openalex.org/W4390099706","doi":"https://doi.org/10.1109/robio58561.2023.10354726"},"language":"en","primary_location":{"id":"doi:10.1109/robio58561.2023.10354726","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/robio58561.2023.10354726","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE International Conference on Robotics and Biomimetics (ROBIO)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5028797524","display_name":"Wing Yin Ng","orcid":"https://orcid.org/0000-0002-4913-8292"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Wing Yin Ng","raw_affiliation_strings":["The Chinese University of Hong Kong,Department of Surgery,Hong Kong","Department of Surgery, The Chinese University of Hong Kong, Hong Kong"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong,Department of Surgery,Hong Kong","institution_ids":["https://openalex.org/I177725633"]},{"raw_affiliation_string":"Department of Surgery, The Chinese University of Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043517197","display_name":"Han Yi Wang","orcid":null},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Han Yi Wang","raw_affiliation_strings":["The Chinese University of Hong Kong,Department of Biomedical Engineering,Hong Kong","Department of Biomedical Engineering, The Chinese University of Hong Kong, Hong Kong"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong,Department of Biomedical Engineering,Hong Kong","institution_ids":["https://openalex.org/I177725633"]},{"raw_affiliation_string":"Department of Biomedical Engineering, The Chinese University of Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5106419004","display_name":"Zheng Li","orcid":"https://orcid.org/0000-0001-9966-1543"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Zheng Li","raw_affiliation_strings":["The Chinese University of Hong Kong,Chow Yuk Ho Technology Centre for Innovative Medicine, Li Ka Shing Institute of Health Science and Multi-Scale Medical Robotics Centre Ltd.,Department of Surgery,Hong Kong","Department of Surgery, Chow Yuk Ho Technology Centre for Innovative Medicine, Li Ka Shing Institute of Health Science and Multi-Scale Medical Robotics Centre Ltd., The Chinese University of Hong Kong, Hong Kong"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong,Chow Yuk Ho Technology Centre for Innovative Medicine, Li Ka Shing Institute of Health Science and Multi-Scale Medical Robotics Centre Ltd.,Department of Surgery,Hong Kong","institution_ids":["https://openalex.org/I177725633"]},{"raw_affiliation_string":"Department of Surgery, Chow Yuk Ho Technology Centre for Innovative Medicine, Li Ka Shing Institute of Health Science and Multi-Scale Medical Robotics Centre Ltd., The Chinese University of Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I177725633"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.35238693,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"3","issue":null,"first_page":"1","last_page":"6"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10830","display_name":"Airway Management and Intubation Techniques","score":0.995199978351593,"subfield":{"id":"https://openalex.org/subfields/2703","display_name":"Anesthesiology and Pain Medicine"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T10830","display_name":"Airway Management and Intubation Techniques","score":0.995199978351593,"subfield":{"id":"https://openalex.org/subfields/2703","display_name":"Anesthesiology and Pain Medicine"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10916","display_name":"Surgical Simulation and Training","score":0.9941999912261963,"subfield":{"id":"https://openalex.org/subfields/2746","display_name":"Surgery"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T11374","display_name":"Cleft Lip and Palate Research","score":0.968500018119812,"subfield":{"id":"https://openalex.org/subfields/1311","display_name":"Genetics"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/minimum-bounding-box","display_name":"Minimum bounding box","score":0.8512818813323975},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7475486397743225},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.729264497756958},{"id":"https://openalex.org/keywords/recall","display_name":"Recall","score":0.6041510105133057},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.4810086190700531},{"id":"https://openalex.org/keywords/identification","display_name":"Identification (biology)","score":0.45267295837402344},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.45199257135391235},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.4426828622817993},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.4043680429458618},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3247266411781311},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.2728549838066101},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.10382586717605591}],"concepts":[{"id":"https://openalex.org/C147037132","wikidata":"https://www.wikidata.org/wiki/Q6865426","display_name":"Minimum bounding box","level":3,"score":0.8512818813323975},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7475486397743225},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.729264497756958},{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.6041510105133057},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.4810086190700531},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.45267295837402344},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.45199257135391235},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.4426828622817993},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.4043680429458618},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3247266411781311},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.2728549838066101},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.10382586717605591},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C59822182","wikidata":"https://www.wikidata.org/wiki/Q441","display_name":"Botany","level":1,"score":0.0},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/robio58561.2023.10354726","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/robio58561.2023.10354726","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE International Conference on Robotics and Biomimetics (ROBIO)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.44999998807907104,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":29,"referenced_works":["https://openalex.org/W180189233","https://openalex.org/W207009764","https://openalex.org/W248357813","https://openalex.org/W639708223","https://openalex.org/W1647671624","https://openalex.org/W1993849402","https://openalex.org/W2023464954","https://openalex.org/W2193145675","https://openalex.org/W2194775991","https://openalex.org/W2279673794","https://openalex.org/W2963163009","https://openalex.org/W2992489396","https://openalex.org/W3106250896","https://openalex.org/W3124517302","https://openalex.org/W3168241213","https://openalex.org/W3172752666","https://openalex.org/W3179888767","https://openalex.org/W3184439416","https://openalex.org/W3196711335","https://openalex.org/W4206340634","https://openalex.org/W4223589100","https://openalex.org/W4225880910","https://openalex.org/W4386076325","https://openalex.org/W6608356501","https://openalex.org/W6636915900","https://openalex.org/W6771289045","https://openalex.org/W6777046832","https://openalex.org/W6798838024","https://openalex.org/W6893711219"],"related_works":["https://openalex.org/W2366107444","https://openalex.org/W2378211422","https://openalex.org/W4388145910","https://openalex.org/W2381570729","https://openalex.org/W1976205134","https://openalex.org/W4321353415","https://openalex.org/W2745001401","https://openalex.org/W4248336175","https://openalex.org/W2031260042","https://openalex.org/W2745033168"],"abstract_inverted_index":{"With":[0],"the":[1,23,51,75,82,114,129,153,162,190,196,199],"increasing":[2],"surgical":[3,15,36,76,83],"need":[4],"in":[5,50,134,202],"our":[6],"aging":[7],"society,":[8],"there":[9],"is":[10],"a":[11,40,177],"lack":[12],"of":[13,25,105,111,125,143,149,168,198],"experienced":[14],"assistants,":[16],"such":[17],"as":[18],"scrub":[19,27,48,68,209],"nurses.":[20,210],"To":[21,170],"facilitate":[22],"training":[24],"junior":[26,67],"nurses":[28,49,69],"and":[29,62,78,93,107,145,206],"to":[30,46,70,127,158,161,184,187],"reduce":[31],"human":[32],"errors,":[33],"e.g.,":[34],"missing":[35],"items,":[37],"we":[38,118,151,175],"develop":[39],"speech-image":[41],"based":[42],"multimodal":[43],"AI":[44],"framework":[45,56,201],"assist":[47],"operating":[52],"room.":[53],"The":[54],"proposed":[55,200],"allows":[57],"real-time":[58,204],"instrument":[59,135],"type":[60],"identification":[61],"instance":[63],"detection,":[64],"which":[65],"enables":[66],"become":[71],"more":[72],"familiar":[73],"with":[74],"instruments":[77],"guides":[79],"them":[80],"throughout":[81],"procedure.":[84,191],"We":[85],"construct":[86],"an":[87,102,108,120,140,146],"ex-vivo":[88],"video-assisted":[89],"thorascopic":[90],"surgery":[91],"dataset":[92],"benchmark":[94],"it":[95],"on":[96,113,189],"common":[97],"object":[98],"detection":[99],"models,":[100],"reaching":[101],"average":[103,109,141,147,154],"precision":[104,142],"98.5%":[106],"recall":[110,148,155],"98.9%":[112],"state-of-the-art":[115],"YOLO-v7.":[116],"Additionally,":[117],"implement":[119],"oriented":[121,164],"bounding":[122,131,165],"box":[123,132,166],"version":[124,167],"YOLO-v7":[126],"address":[128],"undesired":[130],"suppression":[133],"crossing":[136],"over.":[137],"By":[138],"achieving":[139],"95.6%":[144],"97.4%,":[150],"improve":[152],"by":[156],"up":[157],"9.2%":[159],"compared":[160],"previous":[163],"YOLO-v5.":[169],"minimize":[171],"distraction":[172],"during":[173],"surgery,":[174],"adopt":[176],"deep":[178],"learning-based":[179],"automatic":[180],"speech":[181],"recognition":[182],"model":[183],"allow":[185],"surgeons":[186],"concentrate":[188],"Our":[192],"physical":[193],"demonstration":[194],"substantiates":[195],"feasibility":[197],"providing":[203],"guidance":[205],"assistance":[207],"for":[208]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}