{"id":"https://openalex.org/W3034986303","doi":"https://doi.org/10.1145/3397271.3401247","title":"Multi-Level Multimodal Transformer Network for Multimodal Recipe Comprehension","display_name":"Multi-Level Multimodal Transformer Network for Multimodal Recipe Comprehension","publication_year":2020,"publication_date":"2020-07-25","ids":{"openalex":"https://openalex.org/W3034986303","doi":"https://doi.org/10.1145/3397271.3401247","mag":"3034986303"},"language":"en","primary_location":{"id":"doi:10.1145/3397271.3401247","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3397271.3401247","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 43rd International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100346428","display_name":"Ao Liu","orcid":"https://orcid.org/0009-0006-0953-4473"},"institutions":[{"id":"https://openalex.org/I150229711","display_name":"University of Electronic Science and Technology of China","ror":"https://ror.org/04qr3zq92","country_code":"CN","type":"education","lineage":["https://openalex.org/I150229711"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Ao Liu","raw_affiliation_strings":["University of Electronic Science and Technology of China, Chengdu, China"],"affiliations":[{"raw_affiliation_string":"University of Electronic Science and Technology of China, Chengdu, China","institution_ids":["https://openalex.org/I150229711"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101741443","display_name":"Shuai Yuan","orcid":"https://orcid.org/0000-0001-6730-5755"},"institutions":[{"id":"https://openalex.org/I150229711","display_name":"University of Electronic Science and Technology of China","ror":"https://ror.org/04qr3zq92","country_code":"CN","type":"education","lineage":["https://openalex.org/I150229711"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shuai Yuan","raw_affiliation_strings":["University of Electronic Science and Technology of China, Chengdu, China"],"affiliations":[{"raw_affiliation_string":"University of Electronic Science and Technology of China, Chengdu, China","institution_ids":["https://openalex.org/I150229711"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101521207","display_name":"Chenbin Zhang","orcid":"https://orcid.org/0009-0005-9431-9681"},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]},{"id":"https://openalex.org/I150229711","display_name":"University of Electronic Science and Technology of China","ror":"https://ror.org/04qr3zq92","country_code":"CN","type":"education","lineage":["https://openalex.org/I150229711"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chenbin Zhang","raw_affiliation_strings":["University of Electronic Science and Technology of China & Tencent, Chengdu, China"],"affiliations":[{"raw_affiliation_string":"University of Electronic Science and Technology of China & Tencent, Chengdu, China","institution_ids":["https://openalex.org/I2250653659","https://openalex.org/I150229711"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041392292","display_name":"Congjian Luo","orcid":null},"institutions":[{"id":"https://openalex.org/I150229711","display_name":"University of Electronic Science and Technology of China","ror":"https://ror.org/04qr3zq92","country_code":"CN","type":"education","lineage":["https://openalex.org/I150229711"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Congjian Luo","raw_affiliation_strings":["University of Electronic Science and Technology of China & Peng Cheng Lab, Chengdu, China"],"affiliations":[{"raw_affiliation_string":"University of Electronic Science and Technology of China & Peng Cheng Lab, Chengdu, China","institution_ids":["https://openalex.org/I150229711"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079291941","display_name":"Yaqing Liao","orcid":null},"institutions":[{"id":"https://openalex.org/I150229711","display_name":"University of Electronic Science and Technology of China","ror":"https://ror.org/04qr3zq92","country_code":"CN","type":"education","lineage":["https://openalex.org/I150229711"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yaqing Liao","raw_affiliation_strings":["University of Electronic Science and Technology of China, Chengdu, China"],"affiliations":[{"raw_affiliation_string":"University of Electronic Science and Technology of China, Chengdu, China","institution_ids":["https://openalex.org/I150229711"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102906188","display_name":"Kun Bai","orcid":"https://orcid.org/0000-0002-3773-5364"},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kun Bai","raw_affiliation_strings":["Tencent, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Tencent, Beijing, China","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5051227924","display_name":"Zenglin Xu","orcid":"https://orcid.org/0000-0001-5550-6461"},"institutions":[{"id":"https://openalex.org/I150229711","display_name":"University of Electronic Science and Technology of China","ror":"https://ror.org/04qr3zq92","country_code":"CN","type":"education","lineage":["https://openalex.org/I150229711"]},{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]},{"id":"https://openalex.org/I4210136793","display_name":"Peng Cheng Laboratory","ror":"https://ror.org/03qdqbt06","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210136793"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zenglin Xu","raw_affiliation_strings":["Harbin Institute of Technology, Peng Cheng Lab, & University of Electronic Science and Technology of China, Shenzhen, China"],"affiliations":[{"raw_affiliation_string":"Harbin Institute of Technology, Peng Cheng Lab, & University of Electronic Science and Technology of China, Shenzhen, China","institution_ids":["https://openalex.org/I4210136793","https://openalex.org/I204983213","https://openalex.org/I150229711"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5100346428"],"corresponding_institution_ids":["https://openalex.org/I150229711"],"apc_list":null,"apc_paid":null,"fwci":0.2931,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.5545809,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"1781","last_page":"1784"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7697533369064331},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.7373077869415283},{"id":"https://openalex.org/keywords/comprehension","display_name":"Comprehension","score":0.6034042835235596},{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.5871069431304932},{"id":"https://openalex.org/keywords/multimodal-learning","display_name":"Multimodal learning","score":0.5074202418327332},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5009653568267822},{"id":"https://openalex.org/keywords/recipe","display_name":"Recipe","score":0.48222386837005615},{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.472626656293869},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.44811680912971497},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.41500282287597656},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.16675248742103577},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.0702090859413147}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7697533369064331},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.7373077869415283},{"id":"https://openalex.org/C511192102","wikidata":"https://www.wikidata.org/wiki/Q5156948","display_name":"Comprehension","level":2,"score":0.6034042835235596},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.5871069431304932},{"id":"https://openalex.org/C2780660688","wikidata":"https://www.wikidata.org/wiki/Q25052564","display_name":"Multimodal learning","level":2,"score":0.5074202418327332},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5009653568267822},{"id":"https://openalex.org/C2778671685","wikidata":"https://www.wikidata.org/wiki/Q219239","display_name":"Recipe","level":2,"score":0.48222386837005615},{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.472626656293869},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.44811680912971497},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.41500282287597656},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.16675248742103577},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0702090859413147},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C31903555","wikidata":"https://www.wikidata.org/wiki/Q1637030","display_name":"Food science","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3397271.3401247","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3397271.3401247","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 43rd International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Zero hunger","score":0.6200000047683716,"id":"https://metadata.un.org/sdg/2"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":11,"referenced_works":["https://openalex.org/W2194775991","https://openalex.org/W2277195237","https://openalex.org/W2552837936","https://openalex.org/W2737041163","https://openalex.org/W2746097825","https://openalex.org/W2890399523","https://openalex.org/W2963055199","https://openalex.org/W2963403868","https://openalex.org/W2970608575","https://openalex.org/W2987002320","https://openalex.org/W3035260104"],"related_works":["https://openalex.org/W4389505417","https://openalex.org/W4301143707","https://openalex.org/W2952745240","https://openalex.org/W2962931510","https://openalex.org/W4380551887","https://openalex.org/W4285159263","https://openalex.org/W2904518532","https://openalex.org/W4280529741","https://openalex.org/W4293919860","https://openalex.org/W2963650472"],"abstract_inverted_index":{"Multimodal":[0],"Machine":[1],"Comprehension":[2],"($\\rm":[3],"M^3C$)":[4],"has":[5],"been":[6],"a":[7,62],"challenging":[8],"task":[9],"that":[10,104],"requires":[11,35],"understanding":[12],"both":[13],"language":[14],"and":[15,22,70,75,93],"vision,":[16],"as":[17,19,47,49],"well":[18,48],"their":[20],"integration":[21],"interaction.":[23],"For":[24],"example,":[25],"the":[26,50,109,113],"RecipeQA":[27],"challenge,":[28,59],"which":[29],"provides":[30],"several":[31],"$\\rm":[32],"M^3C$":[33],"tasks,":[34],"deep":[36],"neural":[37],"models":[38],"to":[39,68],"understand":[40,71],"textual":[41,73],"instructions,":[42],"images":[43],"of":[44,53,88,98,117],"different":[45,99],"steps,":[46],"logic":[51],"orders":[52],"food":[54],"cooking.":[55],"To":[56],"address":[57],"this":[58],"we":[60],"propose":[61],"Multi-Level":[63],"Multi-Modal":[64],"Transformer":[65],"(MLMM-Trans)":[66],"framework":[67],"integrate":[69],"multiple":[72,76,86],"instructions":[74],"images.":[77],"Our":[78],"model":[79,106],"can":[80,107],"conduct":[81],"intensive":[82],"attention":[83],"mechanism":[84],"at":[85],"levels":[87],"objects":[89],"(e.g.,":[90],"step":[91],"level":[92],"passage-image":[94],"level)":[95],"for":[96],"sequences":[97],"modalities.":[100],"Experiments":[101],"have":[102],"shown":[103],"our":[105],"achieve":[108],"state-of-the-art":[110],"results":[111],"on":[112],"three":[114],"multimodal":[115],"tasks":[116],"RecipeQA.":[118]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2022,"cited_by_count":3}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}