{"id":"https://openalex.org/W4285258389","doi":"https://doi.org/10.1109/jstsp.2022.3171682","title":"Surface Material Perception Through Multimodal Learning","display_name":"Surface Material Perception Through Multimodal Learning","publication_year":2022,"publication_date":"2022-05-11","ids":{"openalex":"https://openalex.org/W4285258389","doi":"https://doi.org/10.1109/jstsp.2022.3171682"},"language":"en","primary_location":{"id":"doi:10.1109/jstsp.2022.3171682","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jstsp.2022.3171682","pdf_url":null,"source":{"id":"https://openalex.org/S42167783","display_name":"IEEE Journal of Selected Topics in Signal Processing","issn_l":"1932-4553","issn":["1932-4553","1941-0484"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal of Selected Topics in Signal Processing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101866586","display_name":"Mao Shi","orcid":"https://orcid.org/0000-0001-9275-4632"},"institutions":[{"id":"https://openalex.org/I4210114105","display_name":"Tsinghua\u2013Berkeley Shenzhen Institute","ror":"https://ror.org/02hhwwz98","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210114105","https://openalex.org/I95457486","https://openalex.org/I99065089"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Shi Mao","raw_affiliation_strings":["Tsinghua-Berkeley Shenzhen Institute and the Department of Electronic Engineering, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-9275-4632","affiliations":[{"raw_affiliation_string":"Tsinghua-Berkeley Shenzhen Institute and the Department of Electronic Engineering, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I4210114105","https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021272610","display_name":"Mengqi Ji","orcid":"https://orcid.org/0000-0002-9210-6851"},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Mengqi Ji","raw_affiliation_strings":["Institute of Artificial Intelligence, Beihang University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Artificial Intelligence, Beihang University, Beijing, China","institution_ids":["https://openalex.org/I82880672"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100372343","display_name":"Bin Wang","orcid":"https://orcid.org/0000-0002-2712-9045"},"institutions":[{"id":"https://openalex.org/I4401727007","display_name":"Hikvision (China)","ror":"https://ror.org/02jzypx27","country_code":null,"type":"company","lineage":["https://openalex.org/I4401727007"]},{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bin Wang","raw_affiliation_strings":["School of Electrical Engineering, Zhejiang University, Hangzhou, China","Network and Information Security Laboratory of Hangzhou Hikvision Digital Technology Company, Ltd., Hangzhou, Zhejiang, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Electrical Engineering, Zhejiang University, Hangzhou, China","institution_ids":["https://openalex.org/I76130692"]},{"raw_affiliation_string":"Network and Information Security Laboratory of Hangzhou Hikvision Digital Technology Company, Ltd., Hangzhou, Zhejiang, China","institution_ids":["https://openalex.org/I4401727007"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080722708","display_name":"Qionghai Dai","orcid":"https://orcid.org/0000-0001-7043-3061"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qionghai Dai","raw_affiliation_strings":["Department of Automation, Tsinghua University, Beijing, China","Institute for Brain and Cognitive Science, Tsinghua University (THUIBCS), Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-7043-3061","affiliations":[{"raw_affiliation_string":"Department of Automation, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"Institute for Brain and Cognitive Science, Tsinghua University (THUIBCS), Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5039161616","display_name":"Lu Fang","orcid":"https://orcid.org/0000-0002-8258-784X"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lu Fang","raw_affiliation_strings":["Department of Electronic Engineering and the Beijing National Research Center for Information Science and Technology, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-8258-784X","affiliations":[{"raw_affiliation_string":"Department of Electronic Engineering and the Beijing National Research Center for Information Science and Technology, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5101866586"],"corresponding_institution_ids":["https://openalex.org/I4210114105","https://openalex.org/I99065089"],"apc_list":null,"apc_paid":null,"fwci":1.8146,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.86076418,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":98},"biblio":{"volume":"16","issue":"4","first_page":"843","last_page":"853"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11211","display_name":"3D Surveying and Cultural Heritage","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/1907","display_name":"Geology"},"field":{"id":"https://openalex.org/fields/19","display_name":"Earth and Planetary Sciences"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11211","display_name":"3D Surveying and Cultural Heritage","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/1907","display_name":"Geology"},"field":{"id":"https://openalex.org/fields/19","display_name":"Earth and Planetary Sciences"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12111","display_name":"Industrial Vision Systems and Defect Detection","score":0.9936000108718872,"subfield":{"id":"https://openalex.org/subfields/2209","display_name":"Industrial and Manufacturing Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11605","display_name":"Visual Attention and Saliency Detection","score":0.9907000064849854,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.7022616863250732},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.6675886511802673},{"id":"https://openalex.org/keywords/rendering","display_name":"Rendering (computer graphics)","score":0.6403615474700928},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5887126922607422},{"id":"https://openalex.org/keywords/structured-light","display_name":"Structured light","score":0.48330211639404297},{"id":"https://openalex.org/keywords/orthogonality","display_name":"Orthogonality","score":0.4555661678314209},{"id":"https://openalex.org/keywords/perception","display_name":"Perception","score":0.41690605878829956},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.4160262942314148},{"id":"https://openalex.org/keywords/geometry","display_name":"Geometry","score":0.2902868092060089},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.2714434564113617}],"concepts":[{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.7022616863250732},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.6675886511802673},{"id":"https://openalex.org/C205711294","wikidata":"https://www.wikidata.org/wiki/Q176953","display_name":"Rendering (computer graphics)","level":2,"score":0.6403615474700928},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5887126922607422},{"id":"https://openalex.org/C193581530","wikidata":"https://www.wikidata.org/wiki/Q683778","display_name":"Structured light","level":2,"score":0.48330211639404297},{"id":"https://openalex.org/C17137986","wikidata":"https://www.wikidata.org/wiki/Q215067","display_name":"Orthogonality","level":2,"score":0.4555661678314209},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.41690605878829956},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.4160262942314148},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.2902868092060089},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2714434564113617},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/jstsp.2022.3171682","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jstsp.2022.3171682","pdf_url":null,"source":{"id":"https://openalex.org/S42167783","display_name":"IEEE Journal of Selected Topics in Signal Processing","issn_l":"1932-4553","issn":["1932-4553","1941-0484"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal of Selected Topics in Signal Processing","raw_type":"journal-article"},{"id":"pmh:oai:repository.hkust.edu.hk:1783.1-157520","is_oa":false,"landing_page_url":"http://repository.hkust.edu.hk/ir/Record/1783.1-157520","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Sustainable cities and communities","id":"https://metadata.un.org/sdg/11","score":0.5400000214576721}],"awards":[{"id":"https://openalex.org/G1126105537","display_name":null,"funder_award_id":"2020M670338","funder_id":"https://openalex.org/F4320321543","funder_display_name":"China Postdoctoral Science Foundation"},{"id":"https://openalex.org/G1940205051","display_name":null,"funder_award_id":"BNR2020RC01002","funder_id":"https://openalex.org/F4320329777","funder_display_name":"Beijing National Research Center For Information Science And Technology"},{"id":"https://openalex.org/G3833898155","display_name":null,"funder_award_id":"62088102","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4236859477","display_name":null,"funder_award_id":"2020TQ0172","funder_id":"https://openalex.org/F4320321543","funder_display_name":"China Postdoctoral Science Foundation"},{"id":"https://openalex.org/G6626028709","display_name":null,"funder_award_id":"YJ20200109","funder_id":"https://openalex.org/F4320321543","funder_display_name":"China Postdoctoral Science Foundation"},{"id":"https://openalex.org/G6779886697","display_name":null,"funder_award_id":"61860206003","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8402275522","display_name":null,"funder_award_id":"62171253","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320321543","display_name":"China Postdoctoral Science Foundation","ror":"https://ror.org/0426zh255"},{"id":"https://openalex.org/F4320329777","display_name":"Beijing National Research Center For Information Science And Technology","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":31,"referenced_works":["https://openalex.org/W1572099491","https://openalex.org/W1953465585","https://openalex.org/W1982402525","https://openalex.org/W2000123870","https://openalex.org/W2006516328","https://openalex.org/W2059552065","https://openalex.org/W2069053895","https://openalex.org/W2075019799","https://openalex.org/W2082086655","https://openalex.org/W2120448006","https://openalex.org/W2149144514","https://openalex.org/W2160642873","https://openalex.org/W2160956336","https://openalex.org/W2163128030","https://openalex.org/W2200948049","https://openalex.org/W2243614535","https://openalex.org/W2402026173","https://openalex.org/W2465446308","https://openalex.org/W2471900868","https://openalex.org/W2486979733","https://openalex.org/W2513285095","https://openalex.org/W2566365295","https://openalex.org/W2608427173","https://openalex.org/W2943940051","https://openalex.org/W2963015618","https://openalex.org/W2963190516","https://openalex.org/W2964339340","https://openalex.org/W3035530517","https://openalex.org/W3087674987","https://openalex.org/W3106186794","https://openalex.org/W3133373188"],"related_works":["https://openalex.org/W2095582735","https://openalex.org/W2059318893","https://openalex.org/W1965698851","https://openalex.org/W834942123","https://openalex.org/W4232542516","https://openalex.org/W1967331680","https://openalex.org/W3176637561","https://openalex.org/W1587967017","https://openalex.org/W130629949","https://openalex.org/W2993249520"],"abstract_inverted_index":{"Accurately":[0],"perceiving":[1],"object":[2],"surface":[3,52],"material":[4,53,80,97,139,159],"is":[5,15,102,136],"critical":[6],"for":[7,51,138],"scene":[8],"understanding":[9],"and":[10,24,35,38,72,75,85,108,144],"robotic":[11],"manipulation.":[12],"However,":[13],"it":[14],"ill-posed":[16],"because":[17],"the":[18,56,105,112,117,124,129,148],"imaging":[19],"process":[20],"entangles":[21],"material,":[22],"lighting,":[23],"geometry":[25,36,91,100],"in":[26,41],"a":[27,46,132],"complex":[28],"way.":[29],"Appearance-based":[30],"methods":[31,165],"cannot":[32],"disentangle":[33,96],"lighting":[34],"variance":[37],"have":[39],"difficulties":[40],"textureless":[42],"regions.":[43],"We":[44],"propose":[45],"novel":[47],"multimodal":[48,133],"fusion":[49,155],"method":[50,135,156],"perception":[54],"using":[55],"depth":[57,113],"camera":[58],"shooting":[59],"structured":[60,118],"laser":[61],"dots.":[62],"The":[63,90,153],"captured":[64,145],"active":[65],"infrared":[66],"image":[67,114],"was":[68],"decomposed":[69],"into":[70],"diffusive":[71],"dot":[73],"modalities":[74],"their":[76],"connection":[77],"with":[78,123],"different":[79],"optical":[81],"properties":[82,98],"(i.e.":[83],"reflection":[84],"scattering)":[86],"were":[87],"revealed":[88],"separately.":[89],"modality,":[92,131],"which":[93],"helps":[94],"to":[95,162],"from":[99,104,116,128],"variations,":[101],"derived":[103],"rendering":[106],"equation":[107],"calculated":[109],"based":[110],"on":[111,142],"obtained":[115],"light":[119],"camera.":[120],"Further,":[121],"together":[122],"texture":[125],"feature":[126],"learned":[127,151],"gray":[130],"learning":[134],"proposed":[137],"perception.":[140],"Experiments":[141],"synthesized":[143],"datasets":[146],"validate":[147],"orthogonality":[149],"of":[150],"features.":[152],"final":[154],"achieves":[157],"92.5%":[158],"accuracy,":[160],"superior":[161],"state-of-the-art":[163],"appearance-based":[164],"(78.4%).":[166]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":3},{"year":2022,"cited_by_count":2}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}