{"id":"https://openalex.org/W7138451707","doi":"https://doi.org/10.1609/aaai.v40i9.37624","title":"IPFormer: Instance Prompt-guided Transformer for Multi-modal Multi-shot Video Understanding","display_name":"IPFormer: Instance Prompt-guided Transformer for Multi-modal Multi-shot Video Understanding","publication_year":2026,"publication_date":"2026-03-14","ids":{"openalex":"https://openalex.org/W7138451707","doi":"https://doi.org/10.1609/aaai.v40i9.37624"},"language":"en","primary_location":{"id":"doi:10.1609/aaai.v40i9.37624","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i9.37624","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://doi.org/10.1609/aaai.v40i9.37624","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5009139809","display_name":"Yujia Liang","orcid":"https://orcid.org/0000-0002-0585-5169"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yujia Liang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006378664","display_name":"Jile Jiao","orcid":"https://orcid.org/0000-0003-2644-717X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jile Jiao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055185546","display_name":"Xuetao Feng","orcid":"https://orcid.org/0000-0002-1309-8954"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xuetao Feng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030704926","display_name":"Xinchen Liu","orcid":"https://orcid.org/0000-0003-4931-8821"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xinchen Liu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129750913","display_name":"Kun Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kun Liu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129721384","display_name":"Yuan WANG","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuan Wang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129668030","display_name":"Zixuan Ye","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zixuan Ye","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129677481","display_name":"Hao Lu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hao Lu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5129750409","display_name":"Zhicheng Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhicheng Wang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":"40","issue":"9","first_page":"6907","last_page":"6915"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9413999915122986,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9413999915122986,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.022700000554323196,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.008899999782443047,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5421000123023987},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.5307999849319458},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.5170000195503235},{"id":"https://openalex.org/keywords/aggregate","display_name":"Aggregate (composite)","score":0.4226999878883362},{"id":"https://openalex.org/keywords/shot","display_name":"Shot (pellet)","score":0.3612000048160553},{"id":"https://openalex.org/keywords/feature-extraction","display_name":"Feature extraction","score":0.3402000069618225},{"id":"https://openalex.org/keywords/simple","display_name":"Simple (philosophy)","score":0.3208000063896179}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8357999920845032},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5421000123023987},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.5307999849319458},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5170000195503235},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.491100013256073},{"id":"https://openalex.org/C4679612","wikidata":"https://www.wikidata.org/wiki/Q866298","display_name":"Aggregate (composite)","level":2,"score":0.4226999878883362},{"id":"https://openalex.org/C2778344882","wikidata":"https://www.wikidata.org/wiki/Q278938","display_name":"Shot (pellet)","level":2,"score":0.3612000048160553},{"id":"https://openalex.org/C52622490","wikidata":"https://www.wikidata.org/wiki/Q1026626","display_name":"Feature extraction","level":2,"score":0.3402000069618225},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3361000120639801},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.33180001378059387},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.3208000063896179},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.3165000081062317},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.30149999260902405},{"id":"https://openalex.org/C2992734406","wikidata":"https://www.wikidata.org/wiki/Q413267","display_name":"One shot","level":2,"score":0.3005000054836273},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.29589998722076416},{"id":"https://openalex.org/C202474056","wikidata":"https://www.wikidata.org/wiki/Q1931635","display_name":"Video tracking","level":3,"score":0.29330000281333923},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2896000146865845},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.257099986076355},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.2563999891281128},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.2529999911785126}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1609/aaai.v40i9.37624","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i9.37624","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},{"id":"pmh:oai:ojs.aaai.org:article/37624","is_oa":false,"landing_page_url":"https://ojs.aaai.org/index.php/AAAI/article/view/37624","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2159-5399","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"doi:10.1609/aaai.v40i9.37624","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i9.37624","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G8582114212","display_name":null,"funder_award_id":"2024YFE0203200","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"}],"funders":[{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Video":[0],"Large":[1],"Language":[2],"Models":[3],"(VideoLLMs),":[4],"which":[5,34,135],"adopt":[6],"large":[7],"language":[8],"models":[9],"for":[10,16,92,185],"video":[11,170,187],"understanding,":[12],"have":[13],"been":[14],"demonstrated":[15],"single-shot":[17],"videos.":[18],"However,":[19],"they":[20],"usually":[21],"struggle":[22],"in":[23],"multi-shot":[24,60,93,169],"videos":[25],"with":[26],"frequent":[27],"shot":[28],"changes,":[29],"varying":[30],"camera":[31],"angles,":[32],"etc.,":[33],"makes":[35],"VideoLLMs":[36,101],"hardly":[37],"answer":[38],"questions":[39],"about":[40],"multiple":[41,155],"instances":[42],"or":[43],"shots":[44],"over":[45],"the":[46,57,68,99,103,123],"whole":[47],"video.":[48],"We":[49],"attribute":[50],"this":[51,147],"challenge":[52],"to":[53,118],"two":[54],"issues:":[55],"1)":[56],"lack":[58],"of":[59,63,70,73,106],"multi-instance":[61,95],"annotations":[62],"existing":[64,100],"datasets,":[65],"and":[66,88,94,165,183],"2)":[67],"negligence":[69],"instance-aware":[71,120,131],"modeling":[72,105],"current":[74],"VideoLLMs.":[75],"Therefore,":[76],"we":[77,109,125],"first":[78],"introduce":[79],"a":[80,111,127],"new":[81],"dataset":[82,164],"termed":[83],"MultiClip-Bench,":[84],"featuring":[85],"dense":[86],"descriptions":[87],"question-answering":[89],"pairs":[90],"tailored":[91],"scenarios.":[96],"Moreover,":[97],"since":[98],"neglect":[102],"explicit":[104],"instance-related":[107],"features,":[108],"propose":[110],"novel":[112],"Instance":[113],"Prompt-guided":[114],"Transformer,":[115],"named":[116],"IPFormer,":[117,124],"achieve":[119],"videounderstanding.":[121],"In":[122],"design":[126],"simple":[128],"but":[129,172],"effective":[130],"feature":[132],"injection":[133],"module,":[134],"encodes":[136],"instance":[137,140],"features":[138],"as":[139],"prompts":[141],"via":[142],"an":[143],"attention-based":[144],"connector.":[145],"By":[146],"means,":[148],"IPFormer":[149],"can":[150,178],"aggregate":[151],"instance-specific":[152],"information":[153],"across":[154],"shots.":[156],"Extensive":[157],"experiments":[158],"not":[159],"only":[160],"show":[161,174],"that":[162,175],"our":[163,176],"model":[166],"significantly":[167],"improve":[168],"understanding.":[171],"also":[173],"MultiClip-Bench":[177],"provide":[179],"valuable":[180],"training":[181],"data":[182],"benchmarks":[184],"various":[186],"understanding":[188],"tasks.":[189]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2026-03-18T00:00:00"}