{"id":"https://openalex.org/W4399116160","doi":"https://doi.org/10.48550/arxiv.2405.16947","title":"Zero-Shot Video Semantic Segmentation based on Pre-Trained Diffusion Models","display_name":"Zero-Shot Video Semantic Segmentation based on Pre-Trained Diffusion Models","publication_year":2024,"publication_date":"2024-05-27","ids":{"openalex":"https://openalex.org/W4399116160","doi":"https://doi.org/10.48550/arxiv.2405.16947"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2405.16947","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.16947","pdf_url":"https://arxiv.org/pdf/2405.16947","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2405.16947","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100391185","display_name":"Qian Wang","orcid":"https://orcid.org/0009-0007-5396-1410"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Qian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033942292","display_name":"Abdelrahman Eldesokey","orcid":"https://orcid.org/0000-0003-3292-7153"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Eldesokey, Abdelrahman","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017462023","display_name":"Mohit Mendiratta","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mendiratta, Mohit","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064046791","display_name":"Fangneng Zhan","orcid":"https://orcid.org/0000-0003-1502-6847"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhan, Fangneng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084928022","display_name":"Adam Kortylewski","orcid":"https://orcid.org/0000-0002-9146-4403"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kortylewski, Adam","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020664641","display_name":"Christian Theobalt","orcid":"https://orcid.org/0000-0001-6104-6625"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Theobalt, Christian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5076768552","display_name":"Peter Wonka","orcid":"https://orcid.org/0000-0003-0627-9746"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wonka, Peter","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11439","display_name":"Video Analysis and Summarization","score":0.9704999923706055,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11439","display_name":"Video Analysis and Summarization","score":0.9704999923706055,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13161","display_name":"Ideological and Political Education","score":0.9526000022888184,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9523000121116638,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/shot","display_name":"Shot (pellet)","score":0.7685296535491943},{"id":"https://openalex.org/keywords/zero","display_name":"Zero (linguistics)","score":0.711794376373291},{"id":"https://openalex.org/keywords/segmentation","display_name":"Segmentation","score":0.6584421396255493},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5834242701530457},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5052455067634583},{"id":"https://openalex.org/keywords/diffusion","display_name":"Diffusion","score":0.4876490831375122},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.38531526923179626},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.3454381227493286},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.08634331822395325},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.08338326215744019},{"id":"https://openalex.org/keywords/materials-science","display_name":"Materials science","score":0.0614224374294281}],"concepts":[{"id":"https://openalex.org/C2778344882","wikidata":"https://www.wikidata.org/wiki/Q278938","display_name":"Shot (pellet)","level":2,"score":0.7685296535491943},{"id":"https://openalex.org/C2780813799","wikidata":"https://www.wikidata.org/wiki/Q3274237","display_name":"Zero (linguistics)","level":2,"score":0.711794376373291},{"id":"https://openalex.org/C89600930","wikidata":"https://www.wikidata.org/wiki/Q1423946","display_name":"Segmentation","level":2,"score":0.6584421396255493},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5834242701530457},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5052455067634583},{"id":"https://openalex.org/C69357855","wikidata":"https://www.wikidata.org/wiki/Q163214","display_name":"Diffusion","level":2,"score":0.4876490831375122},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.38531526923179626},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3454381227493286},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.08634331822395325},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.08338326215744019},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0614224374294281},{"id":"https://openalex.org/C97355855","wikidata":"https://www.wikidata.org/wiki/Q11473","display_name":"Thermodynamics","level":1,"score":0.0},{"id":"https://openalex.org/C191897082","wikidata":"https://www.wikidata.org/wiki/Q11467","display_name":"Metallurgy","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2405.16947","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.16947","pdf_url":"https://arxiv.org/pdf/2405.16947","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2405.16947","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2405.16947","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2405.16947","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.16947","pdf_url":"https://arxiv.org/pdf/2405.16947","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4399116160.pdf","grobid_xml":"https://content.openalex.org/works/W4399116160.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2074502265","https://openalex.org/W4214877189","https://openalex.org/W2773965352","https://openalex.org/W2381179799","https://openalex.org/W2980279061","https://openalex.org/W2334685461","https://openalex.org/W2366718574","https://openalex.org/W2359774528","https://openalex.org/W4298312966","https://openalex.org/W2325697621"],"abstract_inverted_index":{"We":[0,125],"introduce":[1,111,182],"the":[2,39,90,100,134,138,189,193,230],"first":[3],"zero-shot":[4,208],"approach":[5,186,205],"for":[6,115,238],"Video":[7],"Semantic":[8],"Segmentation":[9],"(VSS)":[10],"based":[11,117,132],"on":[12,46,57,83,118,133,214,229],"pre-trained":[13,119],"diffusion":[14,23,123,135],"models.":[15,124],"A":[16],"growing":[17],"research":[18],"direction":[19],"attempts":[20],"to":[21,25,72,85,89,143,145,187,192],"employ":[22],"models":[24],"perform":[26],"downstream":[27],"vision":[28],"tasks":[29,48,59],"by":[30],"exploiting":[31],"their":[32,81],"deep":[33],"understanding":[34],"of":[35,41,92,95],"image":[36,65,120,209],"semantics.":[37],"Yet,":[38],"majority":[40],"these":[42,162],"approaches":[43,68,212,228],"have":[44],"focused":[45],"image-related":[47],"like":[49],"semantic":[50,66,210],"correspondence":[51],"and":[52,110,121],"segmentation,":[53],"with":[54],"less":[55],"emphasis":[56],"video":[58,101,122],"such":[60],"as":[61],"VSS.":[62,239],"Ideally,":[63],"diffusion-based":[64],"segmentation":[67,154,211],"can":[69],"be":[70,86],"applied":[71],"videos":[73,84],"in":[74,99,176],"a":[75,112,128,167,183,197],"frame-by-frame":[76],"manner.":[77],"However,":[78],"we":[79,106,165,181],"find":[80],"performance":[82],"subpar":[87],"due":[88],"absence":[91],"any":[93,219],"modeling":[94],"temporal":[96],"information":[97],"inherent":[98],"data.":[102],"To":[103,160],"this":[104,108],"end,":[105],"tackle":[107],"problem":[109],"framework":[113],"tailored":[114],"VSS":[116,216,227],"propose":[126,166],"building":[127],"scene":[129,146],"context":[130,149],"model":[131,139,150],"features,":[136],"where":[137],"is":[140],"autoregressively":[141],"updated":[142],"adapt":[144],"changes.":[147],"This":[148],"predicts":[151],"per-frame":[152],"coarse":[153,190],"maps":[155,163,191],"that":[156,171,202],"are":[157],"temporally":[158],"consistent.":[159],"refine":[161],"further,":[164],"correspondence-based":[168],"refinement":[169],"strategy":[170],"aggregates":[172],"predictions":[173],"temporally,":[174],"resulting":[175],"more":[177],"confident":[178],"predictions.":[179],"Finally,":[180],"masked":[184],"modulation":[185],"upsample":[188],"full":[194],"resolution":[195],"at":[196],"high":[198],"quality.":[199],"Experiments":[200],"show":[201],"our":[203],"proposed":[204],"outperforms":[206],"existing":[207],"significantly":[213],"various":[215],"benchmarks":[217],"without":[218],"training":[220],"or":[221],"fine-tuning.":[222],"Moreover,":[223],"it":[224],"rivals":[225],"supervised":[226],"VSPW":[231],"dataset":[232],"despite":[233],"not":[234],"being":[235],"explicitly":[236],"trained":[237]},"counts_by_year":[],"updated_date":"2026-08-26T07:47:46.906454","created_date":"2025-10-10T00:00:00"}