{"id":"https://openalex.org/W2998536339","doi":"https://doi.org/10.1609/aaai.v34i05.6294","title":"ManyModalQA: Modality Disambiguation and QA over Diverse Inputs","display_name":"ManyModalQA: Modality Disambiguation and QA over Diverse Inputs","alternate_titles":[],"publication_year":2020,"publication_date":"2020-04-03","ids":{"openalex":"https://openalex.org/W2998536339","doi":"https://doi.org/10.1609/aaai.v34i05.6294","mag":"2998536339"},"language":"en","primary_location":{"id":"doi:10.1609/aaai.v34i05.6294","doi":"https://doi.org/10.1609/aaai.v34i05.6294","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v34i05.6294","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"parent_works":[],"lineage":["https://openalex.org/W2998536339"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://doi.org/10.1609/aaai.v34i05.6294","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5022020152","display_name":"Darryl Hannan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I114027177","display_name":"University of North Carolina at Chapel Hill","ror":"https://ror.org/0130frc33","country_code":"US","type":"education","lineage":["https://openalex.org/I114027177"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Darryl Hannan","raw_affiliation_strings":["UNC Chapel Hill"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UNC Chapel Hill","institution_ids":["https://openalex.org/I114027177"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Akshay Jain","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I114027177","display_name":"University of North Carolina at Chapel Hill","ror":"https://ror.org/0130frc33","country_code":"US","type":"education","lineage":["https://openalex.org/I114027177"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Akshay Jain","raw_affiliation_strings":["UNC Chapel Hill"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UNC Chapel Hill","institution_ids":["https://openalex.org/I114027177"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5001987532","display_name":"Mohit Bansal","orcid":"https://orcid.org/0000-0001-5522-1351","observed_orcids":["https://orcid.org/0000-0001-5522-1351"]},"institutions":[{"id":"https://openalex.org/I114027177","display_name":"University of North Carolina at Chapel Hill","ror":"https://ror.org/0130frc33","country_code":"US","type":"education","lineage":["https://openalex.org/I114027177"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Mohit Bansal","raw_affiliation_strings":["UNC Chapel Hill"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UNC Chapel Hill","institution_ids":["https://openalex.org/I114027177"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I114027177"],"apc_list":null,"apc_paid":null,"fwci":1.936,"has_fulltext":true,"cited_by_count":71,"citation_normalized_percentile":{"value":0.90418691,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":"34","issue":"05","first_page":"7879","last_page":"7886"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10601","display_name":"Handwritten Text Recognition Techniques","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/multimodal-question-answering","display_name":"multimodal question answering","score":0.9760000109672546},{"id":"https://openalex.org/keywords/question-ambiguity","display_name":"question ambiguity","score":0.5619999766349792},{"id":"https://openalex.org/keywords/transfer-learning","display_name":"transfer learning","score":0.6499999761581421}],"concepts":[{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.7792239189147949},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7780094146728516},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.7430500388145447},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.6854245066642761},{"id":"https://openalex.org/C2780522230","wikidata":"https://www.wikidata.org/wiki/Q1140419","display_name":"Ambiguity","level":2,"score":0.6798015832901001},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.6787502765655518},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.6696545481681824},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.599714994430542},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5953744649887085},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5673545002937317},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5254353880882263},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.5249649882316589},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.4667660892009735},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.4508677124977112},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.4316408038139343},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.41426488757133484},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3919001519680023},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0966540277004242},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.06975153088569641},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0},{"id":"https://openalex.org/C111368507","wikidata":"https://www.wikidata.org/wiki/Q43518","display_name":"Oceanography","level":1,"score":0.0},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1609/aaai.v34i05.6294","doi":"https://doi.org/10.1609/aaai.v34i05.6294","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v34i05.6294","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},{"id":"pmh:oai:ojs.aaai.org:article/6294","doi":null,"is_oa":true,"landing_page_url":"https://ojs.aaai.org/index.php/AAAI/article/view/6294","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/6294/6150","source":{"id":"https://openalex.org/S4306400155","display_name":"AAAI Publications (The Association for the Advancement of Artificial Intelligence (AAAI))","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801652627","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/I2801652627"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2159-5399","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"doi:10.1609/aaai.v34i05.6294","doi":"https://doi.org/10.1609/aaai.v34i05.6294","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v34i05.6294","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G7357862116","display_name":null,"funder_award_id":"N66001-19-2-4031","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"},{"id":"https://openalex.org/G7452299184","display_name":null,"funder_award_id":"W911NF","funder_id":"https://openalex.org/F4320338281","funder_display_name":"Army Research Office"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320338281","display_name":"Army Research Office","ror":"https://ror.org/05epdh915"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2998536339.pdf","grobid_xml":"https://content.openalex.org/works/W2998536339.grobid-xml"},"referenced_works_count":41,"referenced_works":["https://openalex.org/W1544827683","https://openalex.org/W1861492603","https://openalex.org/W1933349210","https://openalex.org/W2078238240","https://openalex.org/W2250539671","https://openalex.org/W2282821441","https://openalex.org/W2342096063","https://openalex.org/W2512994153","https://openalex.org/W2550936021","https://openalex.org/W2558203065","https://openalex.org/W2560730294","https://openalex.org/W2561715562","https://openalex.org/W2563399268","https://openalex.org/W2606964149","https://openalex.org/W2606974598","https://openalex.org/W2609826708","https://openalex.org/W2745461083","https://openalex.org/W2746097825","https://openalex.org/W2765383698","https://openalex.org/W2768661419","https://openalex.org/W2773765985","https://openalex.org/W2779681640","https://openalex.org/W2787560479","https://openalex.org/W2794557536","https://openalex.org/W2888296173","https://openalex.org/W2889787757","https://openalex.org/W2890399523","https://openalex.org/W2896457183","https://openalex.org/W2896585994","https://openalex.org/W2951534261","https://openalex.org/W2952228917","https://openalex.org/W2963323070","https://openalex.org/W2963341956","https://openalex.org/W2963420691","https://openalex.org/W2963541336","https://openalex.org/W2963748441","https://openalex.org/W2963866616","https://openalex.org/W2963899988","https://openalex.org/W2965373594","https://openalex.org/W2969862959","https://openalex.org/W2970231061"],"related_works":["https://openalex.org/W3123868215","https://openalex.org/W4389520499","https://openalex.org/W4285249364","https://openalex.org/W4387968448","https://openalex.org/W4385571810","https://openalex.org/W7211995799","https://openalex.org/W4414197602","https://openalex.org/W4415036361","https://openalex.org/W3020743079","https://openalex.org/W4389520156"],"abstract_inverted_index":{"We":[0,26,137],"present":[1],"a":[2,15,68,115,130,144,152,186],"new":[3,145,182],"multimodal":[4,209],"question":[5,16,106],"answering":[6],"challenge,":[7],"ManyModalQA,":[8],"in":[9,45,104,204],"which":[10,102],"an":[11],"agent":[12],"must":[13],"answer":[14,52],"by":[17,30],"considering":[18],"three":[19],"distinct":[20],"modalities:":[21],"text,":[22],"images,":[23],"and":[24,33,74,149,161,193,208],"tables.":[25],"collect":[27,38],"our":[28,82,91,190],"data":[29],"scraping":[31],"Wikipedia":[32],"then":[34],"utilize":[35],"crowdsourcing":[36],"to":[37,86,171],"question-answer":[39],"pairs.":[40],"Our":[41],"questions":[42,92],"are":[43,93,107],"ambiguous,":[44],"that":[46,49,90,158,199],"the":[47,51,60,105,110,123,126,141,156,169],"modality":[48,69,127,206],"contains":[50],"is":[53,185],"not":[54],"easily":[55],"determined":[56],"based":[57,121],"solely":[58],"upon":[59],"question.":[61],"To":[62],"demonstrate":[63],"this":[64,75,98,200],"ambiguity,":[65],"we":[66,100,113,197],"construct":[67,114],"selector":[70],"(or":[71],"disambiguator)":[72],"network,":[73],"model":[76],"gets":[77],"substantially":[78],"lower":[79],"accuracy":[80],"on":[81,122,139],"challenge":[83,201],"set,":[84,154],"compared":[85],"existing":[87,159],"datasets,":[88],"indicating":[89],"more":[94],"ambiguous.":[95],"By":[96],"analyzing":[97],"model,":[99,119],"investigate":[101],"words":[103],"indicative":[108],"of":[109,168],"modality.":[111],"Next,":[112],"simple":[116],"baseline":[117,191],"ManyModalQA":[118],"which,":[120],"prediction":[124],"from":[125],"selector,":[128],"fires":[129],"corresponding":[131],"pre-trained":[132],"state-of-the-art":[133],"unimodal":[134],"QA":[135,210],"model.":[136],"focus":[138],"providing":[140],"community":[142],"with":[143,155],"manymodal":[146],"evaluation":[147],"set":[148],"only":[150],"provide":[151],"fine-tuning":[153],"expectation":[157],"datasets":[160],"approaches":[162],"will":[163],"be":[164],"transferred":[165],"for":[166,180],"most":[167],"training,":[170],"encourage":[172],"low-resource":[173],"generalization":[174],"without":[175],"large,":[176],"monolithic":[177],"training":[178],"sets":[179],"each":[181],"task.":[183],"There":[184],"significant":[187],"gap":[188],"between":[189],"models":[192],"human":[194],"performance;":[195],"therefore,":[196],"hope":[198],"encourages":[202],"research":[203],"end-to-end":[205],"disambiguation":[207],"models,":[211],"as":[212,214],"well":[213],"transfer":[215],"learning.":[216]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W2998536339","counts_by_year":[{"year":2026,"cited_by_count":9},{"year":2025,"cited_by_count":14},{"year":2024,"cited_by_count":18},{"year":2023,"cited_by_count":13},{"year":2022,"cited_by_count":11},{"year":2021,"cited_by_count":6}],"updated_date":"2026-11-21T10:27:49.728292","created_date":"2020-01-10T00:00:00"}
