{"id":"https://openalex.org/W4378771069","doi":"https://doi.org/10.48550/arxiv.2305.18047","title":"InstructEdit: Improving Automatic Masks for Diffusion-based Image Editing With User Instructions","display_name":"InstructEdit: Improving Automatic Masks for Diffusion-based Image Editing With User Instructions","alternate_titles":[],"publication_year":2023,"publication_date":"2023-05-29","ids":{"openalex":"https://openalex.org/W4378771069","doi":"https://doi.org/10.48550/arxiv.2305.18047"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2305.18047","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2305.18047","pdf_url":"https://arxiv.org/pdf/2305.18047","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W4378771069"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2305.18047","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100391185","display_name":"Qian Wang","orcid":"https://orcid.org/0009-0007-5396-1410","observed_orcids":["https://orcid.org/0009-0007-5396-1410"]},"institutions":[{"id":"https://openalex.org/I71920554","display_name":"King Abdullah University of Science and Technology","ror":"https://ror.org/01q3tbs38","country_code":"SA","type":"education","lineage":["https://openalex.org/I71920554"]}],"countries":["SA"],"is_corresponding":false,"raw_author_name":"Wang, Qian","raw_affiliation_strings":["KAUST"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"KAUST","institution_ids":["https://openalex.org/I71920554"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100363706","display_name":"Biao Zhang","orcid":"https://orcid.org/0000-0001-5685-6092","observed_orcids":["https://orcid.org/0000-0001-5685-6092"]},"institutions":[{"id":"https://openalex.org/I71920554","display_name":"King Abdullah University of Science and Technology","ror":"https://ror.org/01q3tbs38","country_code":"SA","type":"education","lineage":["https://openalex.org/I71920554"]}],"countries":["SA"],"is_corresponding":false,"raw_author_name":"Zhang, Biao","raw_affiliation_strings":["KAUST"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"KAUST","institution_ids":["https://openalex.org/I71920554"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028610090","display_name":"Michael Birsak","orcid":"https://orcid.org/0000-0001-6375-8124","observed_orcids":["https://orcid.org/0000-0001-6375-8124"]},"institutions":[{"id":"https://openalex.org/I71920554","display_name":"King Abdullah University of Science and Technology","ror":"https://ror.org/01q3tbs38","country_code":"SA","type":"education","lineage":["https://openalex.org/I71920554"]}],"countries":["SA"],"is_corresponding":false,"raw_author_name":"Birsak, Michael","raw_affiliation_strings":["KAUST"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"KAUST","institution_ids":["https://openalex.org/I71920554"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5076768552","display_name":"Peter Wonka","orcid":"https://orcid.org/0000-0003-0627-9746","observed_orcids":["https://orcid.org/0000-0003-0627-9746"]},"institutions":[{"id":"https://openalex.org/I71920554","display_name":"King Abdullah University of Science and Technology","ror":"https://ror.org/01q3tbs38","country_code":"SA","type":"education","lineage":["https://openalex.org/I71920554"]}],"countries":["SA"],"is_corresponding":false,"raw_author_name":"Wonka, Peter","raw_affiliation_strings":["KAUST"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"KAUST","institution_ids":["https://openalex.org/I71920554"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I71920554"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":90,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12357","display_name":"Digital Media Forensic Detection","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/text-guided-image-editing","display_name":"text-guided image editing","score":0.9430000185966492},{"id":"https://openalex.org/keywords/diffusion-models","display_name":"diffusion models","score":0.8640000224113464},{"id":"https://openalex.org/keywords/user-instructions","display_name":"user instructions","score":0.7300000190734863},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7630000114440918},{"id":"https://openalex.org/keywords/stable-diffusion","display_name":"Stable Diffusion","score":0.8579999804496765}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8806538581848145},{"id":"https://openalex.org/C186644900","wikidata":"https://www.wikidata.org/wiki/Q194152","display_name":"Parsing","level":2,"score":0.7359884977340698},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.6388231515884399},{"id":"https://openalex.org/C89600930","wikidata":"https://www.wikidata.org/wiki/Q1423946","display_name":"Segmentation","level":2,"score":0.6019057631492615},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.5049226880073547},{"id":"https://openalex.org/C2781238097","wikidata":"https://www.wikidata.org/wiki/Q175026","display_name":"Object (grammar)","level":2,"score":0.4700530767440796},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4533608853816986},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4439478814601898},{"id":"https://openalex.org/C124504099","wikidata":"https://www.wikidata.org/wiki/Q56933","display_name":"Image segmentation","level":3,"score":0.4160645306110382},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.38037610054016113},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.370303213596344},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.07944539189338684},{"id":"https://openalex.org/C97355855","wikidata":"https://www.wikidata.org/wiki/Q11473","display_name":"Thermodynamics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:arXiv.org:2305.18047","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2305.18047","pdf_url":"https://arxiv.org/pdf/2305.18047","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:repository.kaust.edu.sa:10754/692507","doi":null,"is_oa":false,"landing_page_url":"http://hdl.handle.net/10754/692507","pdf_url":null,"source":{"id":"https://openalex.org/S4306401596","display_name":"King Abdullah University of Science and Technology Repository (King Abdullah University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I71920554","host_organization_name":"King Abdullah University of Science and Technology","host_organization_lineage":["https://openalex.org/I71920554"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"doi:10.48550/arxiv.2305.18047","doi":"https://doi.org/10.48550/arxiv.2305.18047","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2305.18047","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2305.18047","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2305.18047","pdf_url":"https://arxiv.org/pdf/2305.18047","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4378771069.pdf","grobid_xml":"https://content.openalex.org/works/W4378771069.grobid-xml"},"referenced_works_count":43,"referenced_works":["https://openalex.org/W2962785568","https://openalex.org/W2963341956","https://openalex.org/W2981852735","https://openalex.org/W3030163527","https://openalex.org/W3135367836","https://openalex.org/W3212616005","https://openalex.org/W3216352822","https://openalex.org/W4221146106","https://openalex.org/W4224035735","https://openalex.org/W4226057201","https://openalex.org/W4287029626","https://openalex.org/W4289785045","https://openalex.org/W4300979859","https://openalex.org/W4303440777","https://openalex.org/W4307785224","https://openalex.org/W4308163867","https://openalex.org/W4311034203","https://openalex.org/W4312933868","https://openalex.org/W4318203092","https://openalex.org/W4318718936","https://openalex.org/W4319166718","https://openalex.org/W4321473567","https://openalex.org/W4323927218","https://openalex.org/W4324107312","https://openalex.org/W4324128075","https://openalex.org/W4361865257","https://openalex.org/W4366196888","https://openalex.org/W4367393583","https://openalex.org/W4376122412","https://openalex.org/W4385271281","https://openalex.org/W4385275714","https://openalex.org/W4385527149","https://openalex.org/W4386057725","https://openalex.org/W4386071957","https://openalex.org/W4386076027","https://openalex.org/W4386076215","https://openalex.org/W4386566765","https://openalex.org/W4390873135","https://openalex.org/W4390873195","https://openalex.org/W4390874575","https://openalex.org/W4390874580","https://openalex.org/W4402727429","https://openalex.org/W7133196460"],"related_works":["https://openalex.org/W4318203092","https://openalex.org/W7153106470","https://openalex.org/W4403570987","https://openalex.org/W4408355747","https://openalex.org/W4416075270","https://openalex.org/W4415540740","https://openalex.org/W7133546316","https://openalex.org/W4393156458","https://openalex.org/W4310827167","https://openalex.org/W4401202615"],"abstract_inverted_index":{"Recent":[0],"works":[1],"have":[2],"explored":[3],"text-guided":[4],"image":[5,63,101,150,199],"editing":[6,48,190,194],"using":[7,75],"diffusion":[8],"models":[9,20],"and":[10,30,62,91,97,106,159,174,214],"generated":[11],"edited":[12,29,168,220],"images":[13],"based":[14,49,141],"on":[15,50,142],"text":[16],"prompts.":[17],"However,":[18],"the":[19,25,68,72,88,95,100,115,118,123,143,149,153,156,160,163,167,175,197,209,217,239],"struggle":[21],"to":[22,27,86,135,165],"accurately":[23],"locate":[24],"regions":[26],"be":[28],"faithfully":[31],"perform":[32],"precise":[33],"edits.":[34],"In":[35],"this":[36,83,110,181],"work,":[37],"we":[38],"propose":[39],"a":[40,76,128,138,201],"framework":[41,55,131,227],"termed":[42],"InstructEdit":[43],"that":[44,185,225],"can":[45,228],"do":[46],"fine-grained":[47,193],"user":[51,73,89,233],"instructions.":[52],"Our":[53],"proposed":[54],"has":[56],"three":[57],"components:":[58],"language":[59,69,78,124,157],"processor,":[60,70],"segmenter,":[61,116],"editor.":[64,102],"The":[65,80,112,146],"first":[66],"component,":[67,114,148],"processes":[71],"instruction":[74,90],"large":[77],"model.":[79],"goal":[81],"of":[82,219,232],"processing":[84],"is":[85],"parse":[87],"output":[92],"prompts":[93],"for":[94,99,109,180],"segmenter":[96,164],"captions":[98,154],"We":[103,126,170,207,222,237],"adopt":[104,171],"ChatGPT":[105],"optionally":[107],"BLIP2":[108],"step.":[111],"second":[113],"uses":[117,152],"segmentation":[119,130,144],"prompt":[120],"provided":[121],"by":[122],"processor.":[125],"employ":[127],"state-of-the-art":[129],"Grounded":[132],"Segment":[133],"Anything":[134],"automatically":[136],"generate":[137],"high-quality":[139],"mask":[140,210],"prompt.":[145],"third":[147],"editor,":[151],"from":[155,162,178],"processor":[158],"masks":[161],"compute":[166],"image.":[169],"Stable":[172],"Diffusion":[173],"mask-guided":[176],"generation":[177],"DiffEdit":[179,213],"purpose.":[182],"Experiments":[183],"show":[184,224],"our":[186,226],"method":[187],"outperforms":[188],"previous":[189],"methods":[191],"in":[192],"applications":[195],"where":[196],"input":[198],"contains":[200],"complex":[202],"object":[203],"or":[204],"multiple":[205,230],"objects.":[206],"improve":[208,216],"quality":[211,218],"over":[212],"thus":[215],"images.":[221],"also":[223],"accept":[229],"forms":[231],"instructions":[234],"as":[235],"input.":[236],"provide":[238],"code":[240],"at":[241],"https://github.com/QianWangX/InstructEdit.":[242]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4378771069","counts_by_year":[{"year":2026,"cited_by_count":25},{"year":2025,"cited_by_count":43},{"year":2024,"cited_by_count":18},{"year":2023,"cited_by_count":4}],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2023-05-31T00:00:00"}
