{"id":"https://openalex.org/W2995479498","doi":"https://doi.org/10.48550/arxiv.1912.05783","title":"CLOSURE: Assessing Systematic Generalization of CLEVR Models","display_name":"CLOSURE: Assessing Systematic Generalization of CLEVR Models","publication_year":2019,"publication_date":"2019-12-12","ids":{"openalex":"https://openalex.org/W2995479498","doi":"https://doi.org/10.48550/arxiv.1912.05783","mag":"2995479498"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:1912.05783","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1912.05783","pdf_url":"https://arxiv.org/pdf/1912.05783","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/1912.05783","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5010465328","display_name":"Dzmitry Bahdanau","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I70931966","display_name":"Universit\u00e9 de Montr\u00e9al","ror":"https://ror.org/0161xgx34","country_code":"CA","type":"education","lineage":["https://openalex.org/I70931966"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Bahdanau, Dzmitry","raw_affiliation_strings":["Universit\u00e9 de Montr\u00e9al"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Universit\u00e9 de Montr\u00e9al","institution_ids":["https://openalex.org/I70931966"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103854790","display_name":"Harm de Vries","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I70931966","display_name":"Universit\u00e9 de Montr\u00e9al","ror":"https://ror.org/0161xgx34","country_code":"CA","type":"education","lineage":["https://openalex.org/I70931966"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"de Vries, Harm","raw_affiliation_strings":["Universit\u00e9 de Montr\u00e9al"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Universit\u00e9 de Montr\u00e9al","institution_ids":["https://openalex.org/I70931966"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102014720","display_name":"Timothy J. O\u2019Donnell","orcid":"https://orcid.org/0000-0002-5711-977X","observed_orcids":["https://orcid.org/0000-0002-5711-977X"]},"institutions":[{"id":"https://openalex.org/I5023651","display_name":"McGill University","ror":"https://ror.org/01pxwe438","country_code":"CA","type":"education","lineage":["https://openalex.org/I5023651"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"O'Donnell, Timothy J.","raw_affiliation_strings":["McGill University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"McGill University","institution_ids":["https://openalex.org/I5023651"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054587391","display_name":"Shikhar Murty","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I109736498","display_name":"Canadian Institute for Advanced Research","ror":"https://ror.org/01sdtdd95","country_code":"CA","type":"facility","lineage":["https://openalex.org/I109736498"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["CA","US"],"is_corresponding":false,"raw_author_name":"Murty, Shikhar","raw_affiliation_strings":["CIFAR Fellow","Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CIFAR Fellow","institution_ids":["https://openalex.org/I109736498"]},{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043167556","display_name":"Philippe Beaudoin","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I70931966","display_name":"Universit\u00e9 de Montr\u00e9al","ror":"https://ror.org/0161xgx34","country_code":"CA","type":"education","lineage":["https://openalex.org/I70931966"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Beaudoin, Philippe","raw_affiliation_strings":["Universit\u00e9 de Montr\u00e9al"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Universit\u00e9 de Montr\u00e9al","institution_ids":["https://openalex.org/I70931966"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086198262","display_name":"Yoshua Bengio","orcid":"https://orcid.org/0000-0002-9322-3515","observed_orcids":["https://orcid.org/0000-0002-9322-3515"]},"institutions":[{"id":"https://openalex.org/I4210164802","display_name":"Mila - Quebec Artificial Intelligence Institute","ror":"https://ror.org/05c22rx21","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210164802"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Bengio, Yoshua","raw_affiliation_strings":["Quebec Artificial Intelligence Institute (Mila)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Quebec Artificial Intelligence Institute (Mila)","institution_ids":["https://openalex.org/I4210164802"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5112608251","display_name":"Aaron C. Courville","orcid":"https://orcid.org/0000-0001-6223-0301","observed_orcids":["https://orcid.org/0000-0001-6223-0301"]},"institutions":[{"id":"https://openalex.org/I4210164802","display_name":"Mila - Quebec Artificial Intelligence Institute","ror":"https://ror.org/05c22rx21","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210164802"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Courville, Aaron","raw_affiliation_strings":["Quebec Artificial Intelligence Institute (Mila)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Quebec Artificial Intelligence Institute (Mila)","institution_ids":["https://openalex.org/I4210164802"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":97,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.8988999724388123,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.8988999724388123,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.02070000022649765,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.019600000232458115,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/clevr","display_name":"CLEVR","score":0.7739999890327454},{"id":"https://openalex.org/keywords/systematic-generalization","display_name":"systematic generalization","score":0.9520000219345093},{"id":"https://openalex.org/keywords/referring-expressions","display_name":"referring expressions","score":0.7179999947547913},{"id":"https://openalex.org/keywords/few-shot-transfer-learning","display_name":"few-shot transfer learning","score":0.9330000281333923},{"id":"https://openalex.org/keywords/3d-scene-understanding","display_name":"3D scene understanding","score":0.6710000038146973}],"concepts":[{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.8163387775421143},{"id":"https://openalex.org/C146834321","wikidata":"https://www.wikidata.org/wiki/Q2979672","display_name":"Closure (psychology)","level":2,"score":0.76287442445755},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7227770090103149},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6833792328834534},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.6032472848892212},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5989934206008911},{"id":"https://openalex.org/C53051483","wikidata":"https://www.wikidata.org/wiki/Q861555","display_name":"Cube (algebra)","level":2,"score":0.5027647018432617},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.48091813921928406},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.46851229667663574},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.35297638177871704},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.15102937817573547},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C34447519","wikidata":"https://www.wikidata.org/wiki/Q179522","display_name":"Market economy","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"pmh:oai:arXiv.org:1912.05783","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1912.05783","pdf_url":"https://arxiv.org/pdf/1912.05783","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:2995479498","doi":null,"is_oa":true,"landing_page_url":"http://export.arxiv.org/pdf/1912.05783","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.1912.05783","doi":"https://doi.org/10.48550/arxiv.1912.05783","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1912.05783","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"mag:3013950851","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/pdf/1912.05783.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null}],"best_oa_location":{"id":"pmh:oai:arXiv.org:1912.05783","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1912.05783","pdf_url":"https://arxiv.org/pdf/1912.05783","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2995479498.pdf","grobid_xml":"https://content.openalex.org/works/W2995479498.grobid-xml"},"referenced_works_count":33,"referenced_works":["https://openalex.org/W1933349210","https://openalex.org/W2118373646","https://openalex.org/W2130942839","https://openalex.org/W2194775991","https://openalex.org/W2416885651","https://openalex.org/W2463267937","https://openalex.org/W2563399268","https://openalex.org/W2606220156","https://openalex.org/W2608109911","https://openalex.org/W2613404084","https://openalex.org/W2624614404","https://openalex.org/W2734498959","https://openalex.org/W2760103357","https://openalex.org/W2786209943","https://openalex.org/W2789352267","https://openalex.org/W2793546384","https://openalex.org/W2883104598","https://openalex.org/W2887970879","https://openalex.org/W2891021031","https://openalex.org/W2939413764","https://openalex.org/W2949117887","https://openalex.org/W2950635152","https://openalex.org/W2953212746","https://openalex.org/W2962968135","https://openalex.org/W2963224792","https://openalex.org/W2963267799","https://openalex.org/W2963651499","https://openalex.org/W2964071299","https://openalex.org/W2964121744","https://openalex.org/W2964308564","https://openalex.org/W2996094825","https://openalex.org/W3017374003","https://openalex.org/W3205469723"],"related_works":["https://openalex.org/W7061572011","https://openalex.org/W2907143950","https://openalex.org/W4389519443","https://openalex.org/W4382492135","https://openalex.org/W6901676980","https://openalex.org/W4226080939","https://openalex.org/W2891914492","https://openalex.org/W4416033797","https://openalex.org/W3092750916","https://openalex.org/W4387224648"],"abstract_inverted_index":{"The":[0],"CLEVR":[1,182],"dataset":[2],"of":[3,20,29,35,47,59,63,74,195,204],"natural-looking":[4],"questions":[5],"about":[6],"3D-rendered":[7],"scenes":[8],"has":[9,140],"recently":[10],"received":[11],"much":[12,171],"attention":[13],"from":[14],"the":[15,45,89,93,102,143,151,192,196,205],"research":[16],"community.":[17],"A":[18],"number":[19],"models":[21,49,110,177,197],"have":[22],"been":[23],"proposed":[24],"for":[25],"this":[26,39,68],"task,":[27],"many":[28],"which":[30,54],"achieved":[31],"very":[32],"high":[33],"accuracies":[34],"around":[36],"97-99%.":[37],"In":[38],"work,":[40],"we":[41,70,122,168],"study":[42],"how":[43,170],"systematic":[44,153],"generalization":[46,154],"such":[48],"is,":[50],"that":[51,87,108,124,178,203],"is":[52,88],"to":[53,142,183,185],"extent":[55],"they":[56],"are":[57,179],"capable":[58],"handling":[60],"novel":[61,97,158],"combinations":[62],"known":[64],"linguistic":[65],"constructs.":[66],"To":[67],"end,":[69],"test":[71,147],"models'":[72],"understanding":[73],"referring":[75],"expressions":[76],"based":[77],"on":[78,101,119,135,181],"matching":[79],"object":[80],"properties":[81],"(such":[82],"as":[83,92],"e.g.":[84],"\"another":[85],"cube":[86],"same":[90],"size":[91],"brown":[94],"cube\")":[95],"in":[96],"contexts.":[98],"Our":[99,187],"experiments":[100,190],"thereby":[103],"constructed":[104],"CLOSURE":[105],"benchmark":[106],"show":[107],"state-of-the-art":[109],"often":[111],"do":[112],"not":[113],"exhibit":[114],"systematicity":[115],"after":[116],"being":[117],"trained":[118],"CLEVR.":[120],"Surprisingly,":[121],"find":[123],"an":[125],"explicitly":[126],"compositional":[127],"Neural":[128],"Module":[129],"Network":[130],"model":[131],"also":[132],"generalizes":[133],"badly":[134],"CLOSURE,":[136],"even":[137],"when":[138],"it":[139],"access":[141],"ground-truth":[144],"programs":[145,201],"at":[146],"time.":[148],"We":[149],"improve":[150],"NMN's":[152],"by":[155],"developing":[156],"a":[157],"Vector-NMN":[159],"module":[160],"architecture":[161],"with":[162,198,202],"vector-valued":[163],"inputs":[164],"and":[165],"outputs.":[166],"Lastly,":[167],"investigate":[169],"few-shot":[172,188],"transfer":[173],"learning":[174,189],"can":[175],"help":[176],"pretrained":[180],"adapt":[184],"CLOSURE.":[186],"contrast":[191],"adaptation":[193],"behavior":[194],"intermediate":[199],"discrete":[200],"end-to-end":[206],"continuous":[207],"models.":[208]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":8},{"year":2023,"cited_by_count":33},{"year":2022,"cited_by_count":19},{"year":2021,"cited_by_count":19},{"year":2020,"cited_by_count":9}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2019-12-26T00:00:00"}