{"id":"https://openalex.org/W4403780812","doi":"https://doi.org/10.1145/3664647.3681479","title":"An Entailment Tree Generation Approach for Multimodal Multi-Hop Question Answering with Mixture-of-Experts and Iterative Feedback Mechanism","display_name":"An Entailment Tree Generation Approach for Multimodal Multi-Hop Question Answering with Mixture-of-Experts and Iterative Feedback Mechanism","publication_year":2024,"publication_date":"2024-10-26","ids":{"openalex":"https://openalex.org/W4403780812","doi":"https://doi.org/10.1145/3664647.3681479"},"language":"en","primary_location":{"id":"doi:10.1145/3664647.3681479","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3664647.3681479","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32nd ACM International Conference on Multimedia","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2412.05821","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100358801","display_name":"Qing Zhang","orcid":null},"institutions":[{"id":"https://openalex.org/I1456306","display_name":"North China University of Technology","ror":"https://ror.org/01nky7652","country_code":"CN","type":"education","lineage":["https://openalex.org/I1456306"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Qing Zhang","raw_affiliation_strings":["North China University of Technology, Beijing, China"],"affiliations":[{"raw_affiliation_string":"North China University of Technology, Beijing, China","institution_ids":["https://openalex.org/I1456306"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109667984","display_name":"Haocheng Lv","orcid":null},"institutions":[{"id":"https://openalex.org/I1456306","display_name":"North China University of Technology","ror":"https://ror.org/01nky7652","country_code":"CN","type":"education","lineage":["https://openalex.org/I1456306"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Haocheng Lv","raw_affiliation_strings":["North China University of Technology, Beijing, China"],"affiliations":[{"raw_affiliation_string":"North China University of Technology, Beijing, China","institution_ids":["https://openalex.org/I1456306"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100731156","display_name":"Jie Liu","orcid":"https://orcid.org/0000-0001-5953-4566"},"institutions":[{"id":"https://openalex.org/I1456306","display_name":"North China University of Technology","ror":"https://ror.org/01nky7652","country_code":"CN","type":"education","lineage":["https://openalex.org/I1456306"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jie Liu","raw_affiliation_strings":["North China University of Technology, Beijing, China"],"affiliations":[{"raw_affiliation_string":"North China University of Technology, Beijing, China","institution_ids":["https://openalex.org/I1456306"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062031712","display_name":"Zhiyun Chen","orcid":null},"institutions":[{"id":"https://openalex.org/I1456306","display_name":"North China University of Technology","ror":"https://ror.org/01nky7652","country_code":"CN","type":"education","lineage":["https://openalex.org/I1456306"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhiyun Chen","raw_affiliation_strings":["North China University of Technology, Beijing, China"],"affiliations":[{"raw_affiliation_string":"North China University of Technology, Beijing, China","institution_ids":["https://openalex.org/I1456306"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056722570","display_name":"Jianyong Duan","orcid":"https://orcid.org/0000-0002-2244-3764"},"institutions":[{"id":"https://openalex.org/I1456306","display_name":"North China University of Technology","ror":"https://ror.org/01nky7652","country_code":"CN","type":"education","lineage":["https://openalex.org/I1456306"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jianyong Duan","raw_affiliation_strings":["North China University of Technology, Beijing, China"],"affiliations":[{"raw_affiliation_string":"North China University of Technology, Beijing, China","institution_ids":["https://openalex.org/I1456306"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100599817","display_name":"Hao Wang","orcid":"https://orcid.org/0000-0003-0896-080X"},"institutions":[{"id":"https://openalex.org/I1456306","display_name":"North China University of Technology","ror":"https://ror.org/01nky7652","country_code":"CN","type":"education","lineage":["https://openalex.org/I1456306"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hao Wang","raw_affiliation_strings":["North China University of Technology, Beijing, China"],"affiliations":[{"raw_affiliation_string":"North China University of Technology, Beijing, China","institution_ids":["https://openalex.org/I1456306"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104174131","display_name":"Li He","orcid":"https://orcid.org/0009-0004-7812-6217"},"institutions":[{"id":"https://openalex.org/I1456306","display_name":"North China University of Technology","ror":"https://ror.org/01nky7652","country_code":"CN","type":"education","lineage":["https://openalex.org/I1456306"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Li He","raw_affiliation_strings":["North China University of Technology, Beijing, China"],"affiliations":[{"raw_affiliation_string":"North China University of Technology, Beijing, China","institution_ids":["https://openalex.org/I1456306"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5102686105","display_name":"Mingying Xu","orcid":"https://orcid.org/0009-0004-5018-0270"},"institutions":[{"id":"https://openalex.org/I1456306","display_name":"North China University of Technology","ror":"https://ror.org/01nky7652","country_code":"CN","type":"education","lineage":["https://openalex.org/I1456306"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Mingying Xu","raw_affiliation_strings":["North China University of Technology, Beijing, China"],"affiliations":[{"raw_affiliation_string":"North China University of Technology, Beijing, China","institution_ids":["https://openalex.org/I1456306"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5100358801"],"corresponding_institution_ids":["https://openalex.org/I1456306"],"apc_list":null,"apc_paid":null,"fwci":0.2414,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.53862677,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"4814","last_page":"4822"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9957000017166138,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9868000149726868,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7293548583984375},{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.6220688223838806},{"id":"https://openalex.org/keywords/logical-consequence","display_name":"Logical consequence","score":0.6188571453094482},{"id":"https://openalex.org/keywords/tree","display_name":"Tree (set theory)","score":0.5618626475334167},{"id":"https://openalex.org/keywords/hop","display_name":"Hop (telecommunications)","score":0.5445346236228943},{"id":"https://openalex.org/keywords/mechanism","display_name":"Mechanism (biology)","score":0.49459215998649597},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.46873244643211365},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.1390339434146881},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.06753641366958618}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7293548583984375},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.6220688223838806},{"id":"https://openalex.org/C134752490","wikidata":"https://www.wikidata.org/wiki/Q374182","display_name":"Logical consequence","level":2,"score":0.6188571453094482},{"id":"https://openalex.org/C113174947","wikidata":"https://www.wikidata.org/wiki/Q2859736","display_name":"Tree (set theory)","level":2,"score":0.5618626475334167},{"id":"https://openalex.org/C25906391","wikidata":"https://www.wikidata.org/wiki/Q1432381","display_name":"Hop (telecommunications)","level":2,"score":0.5445346236228943},{"id":"https://openalex.org/C89611455","wikidata":"https://www.wikidata.org/wiki/Q6804646","display_name":"Mechanism (biology)","level":2,"score":0.49459215998649597},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.46873244643211365},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1390339434146881},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.06753641366958618},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3664647.3681479","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3664647.3681479","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32nd ACM International Conference on Multimedia","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2412.05821","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2412.05821","pdf_url":"https://arxiv.org/pdf/2412.05821","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2412.05821","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2412.05821","pdf_url":"https://arxiv.org/pdf/2412.05821","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1121271761","display_name":null,"funder_award_id":"Program","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G1643375234","display_name":null,"funder_award_id":"2024NCUTYXCX102","funder_id":"https://openalex.org/F4320325862","funder_display_name":"North China University of Technology"},{"id":"https://openalex.org/G2087396116","display_name":null,"funder_award_id":"China","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3317480652","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3775455771","display_name":null,"funder_award_id":"U23B2029","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4020255992","display_name":null,"funder_award_id":"Project","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5939423041","display_name":null,"funder_award_id":"Technology","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5994120800","display_name":null,"funder_award_id":"Natural","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G607161736","display_name":null,"funder_award_id":"62076167","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6258663375","display_name":null,"funder_award_id":"2020AAA0109703","funder_id":"https://openalex.org/F4320329860","funder_display_name":"National Science and Technology Major Project"},{"id":"https://openalex.org/G8955107213","display_name":null,"funder_award_id":"Major","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320325862","display_name":"North China University of Technology","ror":"https://ror.org/01nky7652"},{"id":"https://openalex.org/F4320329860","display_name":"National Science and Technology Major Project","ror":null}],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4403780812.pdf"},"referenced_works_count":13,"referenced_works":["https://openalex.org/W1933349210","https://openalex.org/W2809290718","https://openalex.org/W3087931390","https://openalex.org/W3199693760","https://openalex.org/W4205870266","https://openalex.org/W4287855194","https://openalex.org/W4312971273","https://openalex.org/W4385571810","https://openalex.org/W4385573762","https://openalex.org/W4389520499","https://openalex.org/W4390873086","https://openalex.org/W4393147971","https://openalex.org/W4402727764"],"related_works":["https://openalex.org/W2384605597","https://openalex.org/W2387743295","https://openalex.org/W2117210722","https://openalex.org/W2589759689","https://openalex.org/W3082787378","https://openalex.org/W2136007095","https://openalex.org/W4200417912","https://openalex.org/W2971193466","https://openalex.org/W1505456261","https://openalex.org/W1893429831"],"abstract_inverted_index":{"With":[0],"the":[1,30,67,78,83,108,177,180,184,194,202,206],"rise":[2],"of":[3,33,51,160,179,209],"large-scale":[4],"language":[5],"models":[6],"(LLMs),":[7],"it":[8],"is":[9],"currently":[10],"popular":[11],"and":[12,112,125,146,215],"effective":[13],"to":[14,56,63,81,107,169,183,191],"convert":[15],"multimodal":[16,22,115],"information":[17,65],"into":[18],"text":[19],"descriptions":[20],"for":[21,86,186],"multi-hop":[23,35,116],"question":[24,36,117,126],"answering.":[25],"However,":[26],"we":[27,94,130,163],"argue":[28],"that":[29],"current":[31],"methods":[32],"multi-modal":[34],"answering":[37,118,127],"still":[38],"mainly":[39],"face":[40],"two":[41],"challenges:":[42],"1)":[43],"The":[44,70],"retrieved":[45],"evidence":[46],"containing":[47],"a":[48,57,96,120,132,137],"large":[49],"amount":[50],"redundant":[52],"information,":[53],"inevitably":[54],"leads":[55],"significant":[58],"drop":[59],"in":[60,205],"performance":[61],"due":[62,106],"irrelevant":[64],"misleading":[66],"prediction.":[68],"2)":[69],"reasoning":[71,75],"process":[72],"without":[73,101],"interpretable":[74],"steps":[76],"makes":[77],"model":[79],"difficult":[80],"discover":[82],"logical":[84],"errors":[85,152],"handling":[87],"complex":[88],"questions.":[89],"To":[90],"solve":[91],"these":[92],"problems,":[93],"propose":[95],"unified":[97],"LLMs-based":[98],"approach":[99],"but":[100],"heavily":[102],"relying":[103],"on":[104,139,219],"them":[105],"LLM's":[109],"potential":[110,195],"errors,":[111],"innovatively":[113],"treat":[114],"as":[119],"joint":[121,181],"entailment":[122,188],"tree":[123],"generation":[124],"problem.":[128],"Specifically,":[129],"design":[131,164],"multi-task":[133],"learning":[134],"framework":[135],"with":[136,155],"focus":[138],"facilitating":[140],"common":[141],"knowledge":[142],"sharing":[143],"across":[144],"interpretability":[145],"prediction":[147],"tasks":[148,173],"while":[149],"preventing":[150],"task-specific":[151],"from":[153],"interfering":[154],"each":[156],"other":[157],"via":[158],"mixture":[159],"experts.":[161],"Afterward,":[162],"an":[165],"iterative":[166],"feedback":[167],"mechanism":[168],"further":[170],"enhance":[171],"both":[172],"by":[174],"feeding":[175],"back":[176],"results":[178,218],"training":[182],"LLM":[185],"regenerating":[187],"trees,":[189],"aiming":[190],"iteratively":[192],"refine":[193],"answer.":[196],"Notably,":[197],"our":[198],"method":[199],"has":[200],"won":[201],"first":[203],"place":[204],"official":[207],"leaderboard":[208],"WebQA":[210],"(since":[211],"April":[212],"10,":[213],"2024),":[214],"achieves":[216],"competitive":[217],"MultimodalQA.":[220]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-04-18T07:56:08.524223","created_date":"2025-10-10T00:00:00"}