{"id":"https://openalex.org/W3198299822","doi":"https://doi.org/10.1109/iccv48922.2021.00184","title":"Learning to Generate Scene Graph from Natural Language Supervision","display_name":"Learning to Generate Scene Graph from Natural Language Supervision","publication_year":2021,"publication_date":"2021-10-01","ids":{"openalex":"https://openalex.org/W3198299822","doi":"https://doi.org/10.1109/iccv48922.2021.00184","mag":"3198299822"},"language":"en","primary_location":{"id":"doi:10.1109/iccv48922.2021.00184","doi":"https://doi.org/10.1109/iccv48922.2021.00184","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iccv48922.2021.00184","pdf_url":null,"source":{"id":"https://openalex.org/S4363607764","display_name":"IEEE/CVF International Conference on Computer Vision (ICCV)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 IEEE/CVF International Conference on Computer Vision (ICCV)","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2109.02227","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5040199499","display_name":"Yiwu Zhong","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I135310074","display_name":"University of Wisconsin\u2013Madison","ror":"https://ror.org/01y2jtd41","country_code":"US","type":"education","lineage":["https://openalex.org/I135310074"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yiwu Zhong","raw_affiliation_strings":["University of Wisconsin-Madison"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Wisconsin-Madison","institution_ids":["https://openalex.org/I135310074"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037866934","display_name":"Jing Shi","orcid":"https://orcid.org/0000-0001-5624-9698","observed_orcids":["https://orcid.org/0000-0001-5624-9698"]},"institutions":[{"id":"https://openalex.org/I5388228","display_name":"University of Rochester","ror":"https://ror.org/022kthw22","country_code":"US","type":"education","lineage":["https://openalex.org/I5388228"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jing Shi","raw_affiliation_strings":["University of Rochester"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Rochester","institution_ids":["https://openalex.org/I5388228"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101607140","display_name":"Jianwei Yang","orcid":"https://orcid.org/0000-0002-2167-2880","observed_orcids":["https://orcid.org/0000-0002-2167-2880"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jianwei Yang","raw_affiliation_strings":["Microsoft Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064805926","display_name":"Chenliang Xu","orcid":"https://orcid.org/0000-0002-2183-822X","observed_orcids":["https://orcid.org/0000-0002-2183-822X"]},"institutions":[{"id":"https://openalex.org/I5388228","display_name":"University of Rochester","ror":"https://ror.org/022kthw22","country_code":"US","type":"education","lineage":["https://openalex.org/I5388228"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chenliang Xu","raw_affiliation_strings":["University of Rochester"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Rochester","institution_ids":["https://openalex.org/I5388228"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100759456","display_name":"Li Yin","orcid":"https://orcid.org/0000-0001-6769-7947","observed_orcids":["https://orcid.org/0000-0001-6769-7947"]},"institutions":[{"id":"https://openalex.org/I135310074","display_name":"University of Wisconsin\u2013Madison","ror":"https://ror.org/01y2jtd41","country_code":"US","type":"education","lineage":["https://openalex.org/I135310074"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yin Li","raw_affiliation_strings":["University of Wisconsin-Madison"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Wisconsin-Madison","institution_ids":["https://openalex.org/I135310074"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":4.3599,"has_fulltext":true,"cited_by_count":75,"citation_normalized_percentile":{"value":0.9589257,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":88,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1803","last_page":"1814"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scene-graph-generation","display_name":"scene graph generation","score":0.9990000128746033},{"id":"https://openalex.org/keywords/image-text-pairs","display_name":"image-text pairs","score":0.5540000200271606},{"id":"https://openalex.org/keywords/pseudo-labeling","display_name":"pseudo-labeling","score":0.8230000138282776},{"id":"https://openalex.org/keywords/object-detection","display_name":"object detection","score":0.7269999980926514},{"id":"https://openalex.org/keywords/masked-token-prediction","display_name":"masked token prediction","score":0.7300000190734863},{"id":"https://openalex.org/keywords/transformer","display_name":"transformer models","score":0.7509999871253967},{"id":"https://openalex.org/keywords/weakly-supervised-learning","display_name":"weakly supervised learning","score":0.7910000085830688}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7831311225891113},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.7029221057891846},{"id":"https://openalex.org/C179372163","wikidata":"https://www.wikidata.org/wiki/Q1406181","display_name":"Scene graph","level":3,"score":0.6425632238388062},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.513090968132019},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5074445605278015},{"id":"https://openalex.org/C186644900","wikidata":"https://www.wikidata.org/wiki/Q194152","display_name":"Parsing","level":2,"score":0.49820446968078613},{"id":"https://openalex.org/C2777601683","wikidata":"https://www.wikidata.org/wiki/Q6499736","display_name":"Vocabulary","level":2,"score":0.49691513180732727},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.4660787284374237},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.45052486658096313},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.43515947461128235},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.4198063611984253},{"id":"https://openalex.org/C97256817","wikidata":"https://www.wikidata.org/wiki/Q1462316","display_name":"Spurious relationship","level":2,"score":0.4113445580005646},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.4103556275367737},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.30676937103271484},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.14192503690719604},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C205711294","wikidata":"https://www.wikidata.org/wiki/Q176953","display_name":"Rendering (computer graphics)","level":2,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/iccv48922.2021.00184","doi":"https://doi.org/10.1109/iccv48922.2021.00184","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iccv48922.2021.00184","pdf_url":null,"source":{"id":"https://openalex.org/S4363607764","display_name":"IEEE/CVF International Conference on Computer Vision (ICCV)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 IEEE/CVF International Conference on Computer Vision (ICCV)","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2109.02227","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2109.02227","pdf_url":"https://arxiv.org/pdf/2109.02227","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2109.02227","doi":"https://doi.org/10.48550/arxiv.2109.02227","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2109.02227","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2109.02227","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2109.02227","pdf_url":"https://arxiv.org/pdf/2109.02227","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G5926384816","display_name":"RI: Small: Learning Dynamics and Evolution towards Cognitive Understanding of Videos","funder_award_id":"1813709","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320306555","display_name":"Wisconsin Alumni Research Foundation","ror":"https://ror.org/00hwxbz16"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3198299822.pdf","grobid_xml":"https://content.openalex.org/works/W3198299822.grobid-xml"},"referenced_works_count":73,"referenced_works":["https://openalex.org/W639708223","https://openalex.org/W1889081078","https://openalex.org/W1964763677","https://openalex.org/W2080171500","https://openalex.org/W2081580037","https://openalex.org/W2081613070","https://openalex.org/W2098411764","https://openalex.org/W2100031962","https://openalex.org/W2124219775","https://openalex.org/W2134270519","https://openalex.org/W2250378130","https://openalex.org/W2250539671","https://openalex.org/W2277195237","https://openalex.org/W2479423890","https://openalex.org/W2522258376","https://openalex.org/W2579549467","https://openalex.org/W2591644541","https://openalex.org/W2607855566","https://openalex.org/W2626778328","https://openalex.org/W2740962769","https://openalex.org/W2769277182","https://openalex.org/W2810482788","https://openalex.org/W2886641317","https://openalex.org/W2886970679","https://openalex.org/W2890531016","https://openalex.org/W2895682095","https://openalex.org/W2896457183","https://openalex.org/W2899663614","https://openalex.org/W2903371384","https://openalex.org/W2936649169","https://openalex.org/W2952688802","https://openalex.org/W2958447893","https://openalex.org/W2962737704","https://openalex.org/W2962785943","https://openalex.org/W2963101956","https://openalex.org/W2963184176","https://openalex.org/W2963341956","https://openalex.org/W2963649796","https://openalex.org/W2963938081","https://openalex.org/W2963980128","https://openalex.org/W2964323433","https://openalex.org/W2969876226","https://openalex.org/W2970231061","https://openalex.org/W2970608575","https://openalex.org/W2978091520","https://openalex.org/W2981613027","https://openalex.org/W2983943451","https://openalex.org/W2987123286","https://openalex.org/W2990397898","https://openalex.org/W2995460200","https://openalex.org/W2997591391","https://openalex.org/W3015373178","https://openalex.org/W3016121695","https://openalex.org/W3034588855","https://openalex.org/W3034679267","https://openalex.org/W3034727271","https://openalex.org/W3034910302","https://openalex.org/W3035017890","https://openalex.org/W3036253746","https://openalex.org/W3036255508","https://openalex.org/W3037932933","https://openalex.org/W3047398678","https://openalex.org/W3089797718","https://openalex.org/W3090449556","https://openalex.org/W3091588028","https://openalex.org/W3107848485","https://openalex.org/W3109229205","https://openalex.org/W3135367836","https://openalex.org/W3143107425","https://openalex.org/W3173859428","https://openalex.org/W3184370138","https://openalex.org/W3201890160","https://openalex.org/W4288083516"],"related_works":["https://openalex.org/W2963184176","https://openalex.org/W3201591743","https://openalex.org/W2963923113","https://openalex.org/W4386071767","https://openalex.org/W4390872897","https://openalex.org/W2901338647","https://openalex.org/W4386072171","https://openalex.org/W2901525430","https://openalex.org/W7163325300","https://openalex.org/W4414588079"],"abstract_inverted_index":{"Learning":[0,115],"from":[1,38,87,116],"image-text":[2],"data":[3],"has":[4],"demonstrated":[5],"recent":[6],"success":[7],"for":[8,94,142,157,166],"many":[9],"recognition":[10],"tasks,":[11],"yet":[12],"is":[13],"currently":[14],"limited":[15],"to":[16,41,73,84,104],"visual":[17,21],"features":[18],"or":[19],"individual":[20],"concepts":[22,85],"such":[23],"as":[24,56],"objects.":[25],"In":[26,150],"this":[27],"paper,":[28],"we":[29,67,99,152],"propose":[30],"one":[31],"of":[32,46,81],"the":[33,61,163],"first":[34,164],"methods":[35],"that":[36],"learn":[37],"image-sentence":[39,118],"pairs":[40],"extract":[42],"a":[43,101,110,127],"graphical":[44],"representation":[45],"localized":[47],"objects":[48],"and":[49,65,75,89,144,161],"their":[50],"relationships":[51],"within":[52],"an":[53,69,154],"image,":[54],"known":[55],"scene":[57,96,134,147,159,168],"graph.":[58,97],"To":[59],"bridge":[60],"gap":[62],"between":[63],"images":[64],"texts,":[66],"leverage":[68],"off-the-shelf":[70],"object":[71,77],"detector":[72],"identify":[74],"localize":[76],"instances,":[78],"match":[79],"labels":[80,93,108],"detected":[82],"regions":[83],"parsed":[86],"captions,":[88],"thus":[90],"create":[91],"\"pseudo\"":[92,107],"learning":[95],"Further,":[98],"design":[100],"Transformer-based":[102],"model":[103,121,137],"predict":[105],"these":[106],"via":[109],"masked":[111],"token":[112],"prediction":[113],"task.":[114],"only":[117],"pairs,":[119],"our":[120],"achieves":[122],"30%":[123],"relative":[124],"gain":[125],"over":[126],"latest":[128],"method":[129],"trained":[130],"with":[131],"human-annotated":[132],"unlocalized":[133],"graphs.":[135],"Our":[136],"also":[138],"shows":[139],"strong":[140],"results":[141],"weakly":[143],"fully":[145],"supervised":[146],"graph":[148,169],"generation.":[149,170],"addition,":[151],"explore":[153],"open-vocabulary":[155],"setting":[156],"detecting":[158],"graphs,":[160],"present":[162],"result":[165],"open-set":[167]},"counts_by_year":[{"year":2026,"cited_by_count":12},{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":15},{"year":2023,"cited_by_count":25},{"year":2022,"cited_by_count":14},{"year":2021,"cited_by_count":1}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2021-09-13T00:00:00"}