{"id":"https://openalex.org/W2463267937","doi":"https://doi.org/10.18653/v1/d16-1203","title":"Analyzing the Behavior of Visual Question Answering Models","display_name":"Analyzing the Behavior of Visual Question Answering Models","publication_year":2016,"publication_date":"2016-01-01","ids":{"openalex":"https://openalex.org/W2463267937","doi":"https://doi.org/10.18653/v1/d16-1203","mag":"2463267937"},"language":"en","primary_location":{"id":"doi:10.18653/v1/d16-1203","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/d16-1203","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2016 Conference on Empirical Methods in Natural\n          Language Processing","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.18653/v1/d16-1203","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5063960231","display_name":"Aishwarya Agrawal","orcid":"https://orcid.org/0000-0002-8620-8077"},"institutions":[{"id":"https://openalex.org/I859038795","display_name":"Virginia Tech","ror":"https://ror.org/02smfhw86","country_code":"US","type":"education","lineage":["https://openalex.org/I859038795"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aishwarya Agrawal","raw_affiliation_strings":["Virginia Tech, Blacksburg, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Virginia Tech, Blacksburg, United States","institution_ids":["https://openalex.org/I859038795"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014035752","display_name":"Dhruv Batra","orcid":"https://orcid.org/0000-0002-1358-0011"},"institutions":[{"id":"https://openalex.org/I859038795","display_name":"Virginia Tech","ror":"https://ror.org/02smfhw86","country_code":"US","type":"education","lineage":["https://openalex.org/I859038795"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dhruv Batra","raw_affiliation_strings":["Virginia Tech, Blacksburg, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Virginia Tech, Blacksburg, United States","institution_ids":["https://openalex.org/I859038795"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5050342343","display_name":"Devi Parikh","orcid":null},"institutions":[{"id":"https://openalex.org/I859038795","display_name":"Virginia Tech","ror":"https://ror.org/02smfhw86","country_code":"US","type":"education","lineage":["https://openalex.org/I859038795"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Devi Parikh","raw_affiliation_strings":["Virginia Tech, Blacksburg, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Virginia Tech, Blacksburg, United States","institution_ids":["https://openalex.org/I859038795"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I859038795"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":31,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1955","last_page":"1960"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9962000250816345,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.6404505372047424},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6305720806121826},{"id":"https://openalex.org/keywords/strengths-and-weaknesses","display_name":"Strengths and weaknesses","score":0.5993999242782593},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5856717228889465},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5467244386672974},{"id":"https://openalex.org/keywords/active-listening","display_name":"Active listening","score":0.5398411750793457},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4145636558532715},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4112284183502197},{"id":"https://openalex.org/keywords/cognitive-psychology","display_name":"Cognitive psychology","score":0.36399608850479126},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.26600074768066406},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.1342475414276123},{"id":"https://openalex.org/keywords/communication","display_name":"Communication","score":0.06817814707756042},{"id":"https://openalex.org/keywords/economics","display_name":"Economics","score":0.0627855658531189}],"concepts":[{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.6404505372047424},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6305720806121826},{"id":"https://openalex.org/C63882131","wikidata":"https://www.wikidata.org/wiki/Q17122954","display_name":"Strengths and weaknesses","level":2,"score":0.5993999242782593},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5856717228889465},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5467244386672974},{"id":"https://openalex.org/C177291462","wikidata":"https://www.wikidata.org/wiki/Q423038","display_name":"Active listening","level":2,"score":0.5398411750793457},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4145636558532715},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4112284183502197},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.36399608850479126},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.26600074768066406},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.1342475414276123},{"id":"https://openalex.org/C46312422","wikidata":"https://www.wikidata.org/wiki/Q11024","display_name":"Communication","level":1,"score":0.06817814707756042},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0627855658531189},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.18653/v1/d16-1203","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/d16-1203","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2016 Conference on Empirical Methods in Natural\n          Language Processing","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:1606.07356","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1606.07356","pdf_url":"https://arxiv.org/pdf/1606.07356","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"mag:2463267937","is_oa":true,"landing_page_url":"https://www.arxiv.org/pdf/1606.07356","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.1606.07356","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1606.07356","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/v1/d16-1203","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/d16-1203","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2016 Conference on Empirical Methods in Natural\n          Language Processing","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.5699999928474426,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":25,"referenced_works":["https://openalex.org/W1832500336","https://openalex.org/W1951216520","https://openalex.org/W1983927101","https://openalex.org/W2174492417","https://openalex.org/W2190656909","https://openalex.org/W2230472587","https://openalex.org/W2231285690","https://openalex.org/W2256357568","https://openalex.org/W2266930373","https://openalex.org/W2293453011","https://openalex.org/W2340874616","https://openalex.org/W2412393473","https://openalex.org/W2412400526","https://openalex.org/W2439787475","https://openalex.org/W2442626797","https://openalex.org/W2463565445","https://openalex.org/W2475269242","https://openalex.org/W2950577311","https://openalex.org/W2950761309","https://openalex.org/W2951619830","https://openalex.org/W2963191264","https://openalex.org/W2963398599","https://openalex.org/W2963656855","https://openalex.org/W2963890019","https://openalex.org/W2963954913"],"related_works":["https://openalex.org/W2950761309","https://openalex.org/W2560730294","https://openalex.org/W2561715562","https://openalex.org/W2194775991","https://openalex.org/W3037725825","https://openalex.org/W2964065333","https://openalex.org/W2949831469","https://openalex.org/W2472559481","https://openalex.org/W2970017794","https://openalex.org/W2964308564","https://openalex.org/W2963890019","https://openalex.org/W2613718673","https://openalex.org/W2277195237","https://openalex.org/W3094998162","https://openalex.org/W2767171539","https://openalex.org/W2981578638","https://openalex.org/W3171841353","https://openalex.org/W2418349398","https://openalex.org/W3034854924","https://openalex.org/W3109959312"],"abstract_inverted_index":{"Recently,":[0],"a":[1,42,125],"number":[2],"of":[3,13,20,38,69,85,94],"deep-learning":[4],"based":[5],"models":[6,22,40,71,109],"have":[7],"been":[8],"proposed":[9],"for":[10,57],"the":[11,36,53,78,83,91,95,133],"task":[12],"Visual":[14],"Question":[15],"Answering":[16],"(VQA).":[17],"The":[18],"performance":[19],"most":[21,54],"is":[23],"clustered":[24],"around":[25],"60-70%.":[26],"In":[27],"this":[28],"paper":[29],"we":[30],"propose":[31],"systematic":[32],"methods":[33],"to":[34,113,121,130],"analyze":[35,60,90],"behavior":[37,84,100],"these":[39,86],"as":[41],"first":[43],"step":[44],"towards":[45],"recognizing":[46],"their":[47,141],"strengths":[48],"and":[49,51,74,76,80,135],"weaknesses,":[50],"identifying":[52],"fruitful":[55],"directions":[56],"progress.":[58],"We":[59,88],"two":[61,66],"models,":[62],"one":[63],"each":[64],"from":[65],"major":[67],"classes":[68],"VQA":[70,96,108],"--":[72],"with-attention":[73],"without-attention":[75],"show":[77],"similarities":[79],"differences":[81],"in":[82],"models.":[87],"also":[89],"winning":[92],"entry":[93],"Challenge":[97],"2016.":[98],"Our":[99],"analysis":[101],"reveals":[102],"that":[103],"despite":[104],"recent":[105],"progress,":[106],"today's":[107],"are":[110,136],"\"myopic\"":[111],"(tend":[112],"fail":[114],"on":[115,124],"sufficiently":[116],"novel":[117],"instances),":[118],"often":[119],"\"jump":[120],"conclusions\"":[122],"(converge":[123],"predicted":[126],"answer":[127],"after":[128],"'listening'":[129],"just":[131],"half":[132],"question),":[134],"\"stubborn\"":[137],"(do":[138],"not":[139],"change":[140],"answers":[142],"across":[143],"images).":[144]},"counts_by_year":[{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":8},{"year":2020,"cited_by_count":3},{"year":2019,"cited_by_count":4},{"year":2018,"cited_by_count":5},{"year":2017,"cited_by_count":2},{"year":2016,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}