{"id":"https://openalex.org/W4400433702","doi":"https://doi.org/10.48550/arxiv.2407.03471","title":"Learning Action and Reasoning-Centric Image Editing from Videos and Simulations","display_name":"Learning Action and Reasoning-Centric Image Editing from Videos and Simulations","publication_year":2024,"publication_date":"2024-07-03","ids":{"openalex":"https://openalex.org/W4400433702","doi":"https://doi.org/10.48550/arxiv.2407.03471"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2407.03471","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.03471","pdf_url":"https://arxiv.org/pdf/2407.03471","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2407.03471","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5073456064","display_name":"Benno Krojer","orcid":"https://orcid.org/0000-0003-1649-428X","observed_orcids":["https://orcid.org/0000-0003-1649-428X"]},"institutions":[{"id":"https://openalex.org/I5023651","display_name":"McGill University","ror":"https://ror.org/01pxwe438","country_code":"CA","type":"education","lineage":["https://openalex.org/I5023651"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Krojer, Benno","raw_affiliation_strings":["McGill University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"McGill University","institution_ids":["https://openalex.org/I5023651"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092905548","display_name":"Dheeraj Vattikonda","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I5023651","display_name":"McGill University","ror":"https://ror.org/01pxwe438","country_code":"CA","type":"education","lineage":["https://openalex.org/I5023651"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Vattikonda, Dheeraj","raw_affiliation_strings":["Mila & McGill University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mila & McGill University","institution_ids":["https://openalex.org/I5023651"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Lara, Luis","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210164802","display_name":"Mila - Quebec Artificial Intelligence Institute","ror":"https://ror.org/05c22rx21","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210164802"]},{"id":"https://openalex.org/I5023651","display_name":"McGill University","ror":"https://ror.org/01pxwe438","country_code":"CA","type":"education","lineage":["https://openalex.org/I5023651"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Lara, Luis","raw_affiliation_strings":["Mila & McGill University Facebook CIFAR AI Chair ServiceNow Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mila & McGill University Facebook CIFAR AI Chair ServiceNow Research","institution_ids":["https://openalex.org/I4210164802","https://openalex.org/I5023651"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085870393","display_name":"Varun Jampani","orcid":"https://orcid.org/0000-0002-1508-6498","observed_orcids":["https://orcid.org/0000-0002-1508-6498"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jampani, Varun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001944487","display_name":"Eva Portelance","orcid":"https://orcid.org/0009-0007-7762-8516","observed_orcids":["https://orcid.org/0009-0007-7762-8516"]},"institutions":[{"id":"https://openalex.org/I108192572","display_name":"HEC Montr\u00e9al","ror":"https://ror.org/05ww3wq27","country_code":"CA","type":"education","lineage":["https://openalex.org/I108192572"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Portelance, Eva","raw_affiliation_strings":["HEC Montr\u00e9al"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"HEC Montr\u00e9al","institution_ids":["https://openalex.org/I108192572"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075885606","display_name":"Christopher Pal","orcid":"https://orcid.org/0000-0001-6534-2114","observed_orcids":["https://orcid.org/0000-0001-6534-2114"]},"institutions":[{"id":"https://openalex.org/I109736498","display_name":"Canadian Institute for Advanced Research","ror":"https://ror.org/01sdtdd95","country_code":"CA","type":"facility","lineage":["https://openalex.org/I109736498"]},{"id":"https://openalex.org/I4210164802","display_name":"Mila - Quebec Artificial Intelligence Institute","ror":"https://ror.org/05c22rx21","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210164802"]},{"id":"https://openalex.org/I45683168","display_name":"Polytechnique Montr\u00e9al","ror":"https://ror.org/05f8d4e86","country_code":"CA","type":"education","lineage":["https://openalex.org/I45683168"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Pal, Christopher","raw_affiliation_strings":["Mila & Polytechnique Montr\u00e9al Canada CIFAR AI Chair ServiceNow Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mila & Polytechnique Montr\u00e9al Canada CIFAR AI Chair ServiceNow Research","institution_ids":["https://openalex.org/I109736498","https://openalex.org/I4210164802","https://openalex.org/I45683168"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5102886212","display_name":"Siva Reddy","orcid":"https://orcid.org/0000-0003-3753-0323","observed_orcids":["https://orcid.org/0000-0003-3753-0323"]},"institutions":[{"id":"https://openalex.org/I4210164802","display_name":"Mila - Quebec Artificial Intelligence Institute","ror":"https://ror.org/05c22rx21","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210164802"]},{"id":"https://openalex.org/I5023651","display_name":"McGill University","ror":"https://ror.org/01pxwe438","country_code":"CA","type":"education","lineage":["https://openalex.org/I5023651"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Reddy, Siva","raw_affiliation_strings":["Mila & McGill University Facebook CIFAR AI Chair ServiceNow Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mila & McGill University Facebook CIFAR AI Chair ServiceNow Research","institution_ids":["https://openalex.org/I4210164802","https://openalex.org/I5023651"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":9,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12357","display_name":"Digital Media Forensic Detection","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/image-editing","display_name":"image editing","score":0.7860000133514404},{"id":"https://openalex.org/keywords/training-data-curation","display_name":"training data curation","score":0.7990000247955322},{"id":"https://openalex.org/keywords/automatic-evaluation-metrics","display_name":"automatic evaluation metrics","score":0.7559999823570251},{"id":"https://openalex.org/keywords/object-replacement","display_name":"object replacement","score":0.4569999873638153},{"id":"https://openalex.org/keywords/attribute-editing","display_name":"attribute editing","score":0.5479999780654907},{"id":"https://openalex.org/keywords/physical-dynamics","display_name":"physical dynamics","score":0.35100001096725464}],"concepts":[{"id":"https://openalex.org/C2776674983","wikidata":"https://www.wikidata.org/wiki/Q545981","display_name":"Image editing","level":3,"score":0.6996810436248779},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.648220419883728},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6128625869750977},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.5266774892807007},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4755782186985016},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.44543129205703735},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.4074605107307434},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.34644782543182373},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.19103893637657166},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2407.03471","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.03471","pdf_url":"https://arxiv.org/pdf/2407.03471","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2407.03471","doi":"https://doi.org/10.48550/arxiv.2407.03471","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2407.03471","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2407.03471","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.03471","pdf_url":"https://arxiv.org/pdf/2407.03471","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4400433702.pdf","grobid_xml":"https://content.openalex.org/works/W4400433702.grobid-xml"},"referenced_works_count":14,"referenced_works":["https://openalex.org/W3203737321","https://openalex.org/W4224035735","https://openalex.org/W4285265603","https://openalex.org/W4353113081","https://openalex.org/W4362707108","https://openalex.org/W4385569875","https://openalex.org/W4390872842","https://openalex.org/W4394868314","https://openalex.org/W4394972242","https://openalex.org/W4402671992","https://openalex.org/W4402703020","https://openalex.org/W4402753379","https://openalex.org/W4407450948","https://openalex.org/W4412636516"],"related_works":["https://openalex.org/W4415945551","https://openalex.org/W4414588487","https://openalex.org/W7221174045","https://openalex.org/W4398191746","https://openalex.org/W7084052711","https://openalex.org/W4416452763","https://openalex.org/W7161180978","https://openalex.org/W4404404778","https://openalex.org/W4417009938","https://openalex.org/W7163708704"],"abstract_inverted_index":{"An":[0],"image":[1,242],"editing":[2,33,165,172,196],"model":[3,155,168],"should":[4],"be":[5,49],"able":[6],"to":[7,19,70],"perform":[8],"diverse":[9,164],"edits,":[10],"ranging":[11],"from":[12,51,72,104],"object":[13],"replacement,":[14],"changing":[15],"attributes":[16],"or":[17,22,45],"style,":[18],"performing":[20],"actions":[21],"movement,":[23],"which":[24],"require":[25],"many":[26],"forms":[27],"of":[28,97,115,148],"reasoning.":[29,84],"Current":[30],"general":[31,241],"instruction-guided":[32],"models":[34,173],"have":[35],"significant":[36],"shortcomings":[37],"with":[38],"action":[39,62],"and":[40,63,68,82,102,106,141,189,222,230],"reasoning-centric":[41,64],"edits.":[42],"Object,":[43],"attribute":[44],"stylistic":[46],"changes":[47,138],"can":[48],"learned":[50],"visually":[52],"static":[53],"datasets.":[54],"On":[55],"the":[56,91,133,146],"other":[57],"hand,":[58],"high-quality":[59,98],"data":[60],"for":[61,193],"edits":[65],"is":[66],"scarce":[67],"has":[69],"come":[71],"entirely":[73],"different":[74],"sources":[75],"that":[76,205,212],"cover":[77],"e.g.":[78],"physical":[79],"dynamics,":[80],"temporality":[81],"spatial":[83],"To":[85,144],"this":[86],"end,":[87],"we":[88,151,182,199],"meticulously":[89],"curate":[90],"AURORA":[92],"Dataset":[93],"(Action-Reasoning-Object-Attribute),":[94],"a":[95,112,126,157,201,218,233],"collection":[96],"training":[99,117,220],"data,":[100],"human-annotated":[101],"curated":[103],"videos":[105],"simulation":[107],"engines.":[108],"We":[109,210],"focus":[110],"on":[111,156,207,240],"key":[113],"aspect":[114],"quality":[116,219],"data:":[118],"triplets":[119],"(source":[120],"image,":[121],"prompt,":[122,134],"target":[123,142],"image)":[124],"contain":[125],"single":[127],"meaningful":[128],"visual":[129],"change":[130],"described":[131],"by":[132,176],"i.e.,":[135],"truly":[136],"minimal":[137],"between":[139],"source":[140],"images.":[143],"demonstrate":[145],"value":[147],"our":[149,213],"dataset,":[150],"evaluate":[152],"an":[153,223],"AURORA-finetuned":[154],"new":[158,202],"expert-curated":[159],"benchmark":[160],"(AURORA-Bench)":[161],"covering":[162],"8":[163],"tasks.":[166,197],"Our":[167],"significantly":[169],"outperforms":[170],"previous":[171,187],"as":[174],"judged":[175],"human":[177],"raters.":[178],"For":[179],"automatic":[180,203],"evaluations,":[181,229],"find":[183],"important":[184],"flaws":[185],"in":[186],"metrics":[188],"caution":[190],"their":[191],"use":[192],"semantically":[194],"hard":[195],"Instead,":[198],"propose":[200],"metric":[204],"focuses":[206],"discriminative":[208],"understanding.":[209],"hope":[211],"efforts":[214],":":[215],"(1)":[216],"curating":[217],"dataset":[221],"evaluation":[224],"benchmark,":[225],"(2)":[226],"developing":[227],"critical":[228],"(3)":[231],"releasing":[232],"state-of-the-art":[234],"model,":[235],"will":[236],"fuel":[237],"further":[238],"progress":[239],"editing.":[243]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":5}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2024-07-09T00:00:00"}