{"id":"https://openalex.org/W4403791857","doi":"https://doi.org/10.1145/3664647.3681275","title":"Multimodal Inplace Prompt Tuning for Open-set Object Detection","display_name":"Multimodal Inplace Prompt Tuning for Open-set Object Detection","publication_year":2024,"publication_date":"2024-10-26","ids":{"openalex":"https://openalex.org/W4403791857","doi":"https://doi.org/10.1145/3664647.3681275"},"language":"en","primary_location":{"id":"doi:10.1145/3664647.3681275","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3664647.3681275","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32nd ACM International Conference on Multimedia","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5090010368","display_name":"Guilin Li","orcid":"https://orcid.org/0009-0000-6885-0261"},"institutions":[{"id":"https://openalex.org/I191208505","display_name":"Xiamen University","ror":"https://ror.org/00mcjh785","country_code":"CN","type":"education","lineage":["https://openalex.org/I191208505"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Guilin Li","raw_affiliation_strings":["Key Laboratory of Multimedia Trusted Perception and Efficient Computing, Ministry of Education of China, Xiamen University, xiamen, China"],"affiliations":[{"raw_affiliation_string":"Key Laboratory of Multimedia Trusted Perception and Efficient Computing, Ministry of Education of China, Xiamen University, xiamen, China","institution_ids":["https://openalex.org/I191208505"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084838057","display_name":"Mengdan Zhang","orcid":null},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Mengdan Zhang","raw_affiliation_strings":["Tencent Youtu Lab, shanghai, China"],"affiliations":[{"raw_affiliation_string":"Tencent Youtu Lab, shanghai, China","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054226277","display_name":"Xiawu Zheng","orcid":"https://orcid.org/0000-0002-6855-5403"},"institutions":[{"id":"https://openalex.org/I191208505","display_name":"Xiamen University","ror":"https://ror.org/00mcjh785","country_code":"CN","type":"education","lineage":["https://openalex.org/I191208505"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiawu Zheng","raw_affiliation_strings":["Key Laboratory of Multimedia Trusted Perception and Efficient Computing, Ministry of Education of China, Xiamen University, xiamen, China"],"affiliations":[{"raw_affiliation_string":"Key Laboratory of Multimedia Trusted Perception and Efficient Computing, Ministry of Education of China, Xiamen University, xiamen, China","institution_ids":["https://openalex.org/I191208505"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109924324","display_name":"Peixian Chen","orcid":"https://orcid.org/0009-0008-8015-5366"},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Peixian Chen","raw_affiliation_strings":["Tencent YouTu Lab, shanghai, China"],"affiliations":[{"raw_affiliation_string":"Tencent YouTu Lab, shanghai, China","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019172013","display_name":"Zihan Wang","orcid":"https://orcid.org/0009-0000-5184-6637"},"institutions":[{"id":"https://openalex.org/I66867065","display_name":"East China Normal University","ror":"https://ror.org/02n96ep67","country_code":"CN","type":"education","lineage":["https://openalex.org/I66867065"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zihan Wang","raw_affiliation_strings":["East China Normal University, shanghai, China"],"affiliations":[{"raw_affiliation_string":"East China Normal University, shanghai, China","institution_ids":["https://openalex.org/I66867065"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039883116","display_name":"Yunhang Shen","orcid":"https://orcid.org/0000-0002-3970-7519"},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yunhang Shen","raw_affiliation_strings":["Tencent YouTu Lab, shanghai, China"],"affiliations":[{"raw_affiliation_string":"Tencent YouTu Lab, shanghai, China","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030077358","display_name":"Mingchen Zhuge","orcid":"https://orcid.org/0000-0003-2561-7712"},"institutions":[{"id":"https://openalex.org/I71920554","display_name":"King Abdullah University of Science and Technology","ror":"https://ror.org/01q3tbs38","country_code":"SA","type":"education","lineage":["https://openalex.org/I71920554"]}],"countries":["SA"],"is_corresponding":false,"raw_author_name":"Mingchen Zhuge","raw_affiliation_strings":["King Abdullah University of Science and Technology, thuwal, Saudi Arabia"],"affiliations":[{"raw_affiliation_string":"King Abdullah University of Science and Technology, thuwal, Saudi Arabia","institution_ids":["https://openalex.org/I71920554"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101513277","display_name":"Chenglin Wu","orcid":"https://orcid.org/0000-0002-5035-3168"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chenglin Wu","raw_affiliation_strings":["DeepWisdom, xiamen, China"],"affiliations":[{"raw_affiliation_string":"DeepWisdom, xiamen, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000389309","display_name":"Fei Chao","orcid":"https://orcid.org/0000-0002-6928-2638"},"institutions":[{"id":"https://openalex.org/I191208505","display_name":"Xiamen University","ror":"https://ror.org/00mcjh785","country_code":"CN","type":"education","lineage":["https://openalex.org/I191208505"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fei Chao","raw_affiliation_strings":["Key Laboratory of Multimedia Trusted Perception and Efficient Computing, Ministry of Education of China, Xiamen University, xiamen, China"],"affiliations":[{"raw_affiliation_string":"Key Laboratory of Multimedia Trusted Perception and Efficient Computing, Ministry of Education of China, Xiamen University, xiamen, China","institution_ids":["https://openalex.org/I191208505"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100725160","display_name":"Ke Li","orcid":"https://orcid.org/0000-0001-7998-0731"},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ke Li","raw_affiliation_strings":["Tencent YouTu Lab, shanghai, China"],"affiliations":[{"raw_affiliation_string":"Tencent YouTu Lab, shanghai, China","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004402130","display_name":"Xing Sun","orcid":"https://orcid.org/0000-0001-8132-9083"},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xing Sun","raw_affiliation_strings":["Tencent YouTu Lab, shanghai, China"],"affiliations":[{"raw_affiliation_string":"Tencent YouTu Lab, shanghai, China","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5016080094","display_name":"Rongrong Ji","orcid":"https://orcid.org/0000-0001-9163-2932"},"institutions":[{"id":"https://openalex.org/I191208505","display_name":"Xiamen University","ror":"https://ror.org/00mcjh785","country_code":"CN","type":"education","lineage":["https://openalex.org/I191208505"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Rongrong Ji","raw_affiliation_strings":["Key Laboratory of Multimedia Trusted Perception and Efficient Computing, Ministry of Education of China, Xiamen University, xiamen, China"],"affiliations":[{"raw_affiliation_string":"Key Laboratory of Multimedia Trusted Perception and Efficient Computing, Ministry of Education of China, Xiamen University, xiamen, China","institution_ids":["https://openalex.org/I191208505"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":12,"corresponding_author_ids":["https://openalex.org/A5090010368"],"corresponding_institution_ids":["https://openalex.org/I191208505"],"apc_list":null,"apc_paid":null,"fwci":0.5054,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.66414956,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"8062","last_page":"8071"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10331","display_name":"Video Surveillance and Tracking Methods","score":0.9975000023841858,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7159332036972046},{"id":"https://openalex.org/keywords/object-detection","display_name":"Object detection","score":0.5468778014183044},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.4822300374507904},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.48025229573249817},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4624856114387512},{"id":"https://openalex.org/keywords/open-set","display_name":"Open set","score":0.4298502802848816},{"id":"https://openalex.org/keywords/object","display_name":"Object (grammar)","score":0.42554229497909546},{"id":"https://openalex.org/keywords/object-class-detection","display_name":"Object-class detection","score":0.4206992983818054},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.2176268994808197},{"id":"https://openalex.org/keywords/face-detection","display_name":"Face detection","score":0.16774457693099976},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.12083578109741211},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.10146161913871765}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7159332036972046},{"id":"https://openalex.org/C2776151529","wikidata":"https://www.wikidata.org/wiki/Q3045304","display_name":"Object detection","level":3,"score":0.5468778014183044},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4822300374507904},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.48025229573249817},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4624856114387512},{"id":"https://openalex.org/C42357961","wikidata":"https://www.wikidata.org/wiki/Q213363","display_name":"Open set","level":2,"score":0.4298502802848816},{"id":"https://openalex.org/C2781238097","wikidata":"https://www.wikidata.org/wiki/Q175026","display_name":"Object (grammar)","level":2,"score":0.42554229497909546},{"id":"https://openalex.org/C71681937","wikidata":"https://www.wikidata.org/wiki/Q3045304","display_name":"Object-class detection","level":5,"score":0.4206992983818054},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.2176268994808197},{"id":"https://openalex.org/C4641261","wikidata":"https://www.wikidata.org/wiki/Q11681085","display_name":"Face detection","level":4,"score":0.16774457693099976},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.12083578109741211},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.10146161913871765},{"id":"https://openalex.org/C31510193","wikidata":"https://www.wikidata.org/wiki/Q1192553","display_name":"Facial recognition system","level":3,"score":0.0},{"id":"https://openalex.org/C118615104","wikidata":"https://www.wikidata.org/wiki/Q121416","display_name":"Discrete mathematics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3664647.3681275","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3664647.3681275","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32nd ACM International Conference on Multimedia","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":38,"referenced_works":["https://openalex.org/W1536680647","https://openalex.org/W2194775991","https://openalex.org/W2565639579","https://openalex.org/W2948672349","https://openalex.org/W2963037989","https://openalex.org/W2963936013","https://openalex.org/W2983943451","https://openalex.org/W2997998901","https://openalex.org/W2998356391","https://openalex.org/W3138516171","https://openalex.org/W3159619744","https://openalex.org/W3166140588","https://openalex.org/W3173859428","https://openalex.org/W4212774754","https://openalex.org/W4224246420","https://openalex.org/W4282919422","https://openalex.org/W4287122891","https://openalex.org/W4309181071","https://openalex.org/W4312310776","https://openalex.org/W4312424618","https://openalex.org/W4312563428","https://openalex.org/W4312747482","https://openalex.org/W4312884055","https://openalex.org/W4312956471","https://openalex.org/W4313026212","https://openalex.org/W4318954855","https://openalex.org/W4386065815","https://openalex.org/W4386065936","https://openalex.org/W4386066126","https://openalex.org/W4386075561","https://openalex.org/W4386076029","https://openalex.org/W4386076396","https://openalex.org/W4386790226","https://openalex.org/W4390873107","https://openalex.org/W4402672034","https://openalex.org/W4402728160","https://openalex.org/W4402780269","https://openalex.org/W4403725720"],"related_works":["https://openalex.org/W2901758161","https://openalex.org/W4205668735","https://openalex.org/W3204852000","https://openalex.org/W3126664501","https://openalex.org/W3209429418","https://openalex.org/W4297540035","https://openalex.org/W4292830139","https://openalex.org/W4319309705","https://openalex.org/W4312834249","https://openalex.org/W2997780656"],"abstract_inverted_index":{"The":[0],"integration":[1],"of":[2,68,116],"large":[3,69],"language":[4,45,70],"models":[5,21,46,71],"into":[6],"open-world":[7,33],"detection":[8,48],"frameworks":[9],"significantly":[10],"improves":[11],"versatility":[12],"in":[13,47,51,91],"new":[14],"environments.":[15],"Prompt":[16],"representations":[17],"derived":[18],"from":[19,129],"these":[20],"help":[22],"establish":[23],"classification":[24],"boundaries":[25],"for":[26,76],"both":[27],"base":[28],"and":[29,55,72,135],"novel":[30],"categories":[31],"within":[32],"detectors.":[34],"However,":[35],"we":[36,96],"are":[37],"the":[38,66,105,114],"first":[39],"to":[40,57,63,85,103,131],"discover":[41],"that":[42],"directly":[43],"fine-tuning":[44],"systems":[49],"results":[50,112],"redundant":[52],"attention":[53,88],"patterns":[54],"leads":[56],"suboptimal":[58],"prompt":[59,74,100,107],"representations.":[60],"In":[61],"order":[62],"fully":[64],"leverage":[65],"capabilities":[67],"augment":[73],"encoding":[75],"detection,":[77],"this":[78],"study":[79],"introduces":[80],"a":[81,121],"redundancy":[82],"assessment":[83],"metric":[84],"identify":[86],"uniform":[87],"patterns.":[89],"Furthermore,":[90],"areas":[92],"with":[93,108],"high":[94],"redundancy,":[95],"incorporate":[97],"multimodal":[98],"inplace":[99],"tuning":[101],"(MIPT)":[102],"enrich":[104],"text":[106],"visual":[109],"clues.":[110],"Experimental":[111],"validate":[113],"efficacy":[115],"our":[117],"MIPT":[118],"framework,":[119],"achieving":[120],"notable":[122],"increase":[123],"across":[124],"benchmarks,":[125],"e.g.":[126],"elevating":[127],"GLIP-L":[128],"22.6%":[130],"25.0%":[132],"on":[133,138],"ODinW-35,":[134],"9.0%":[136],"improvement":[137],"LVIS.":[139]},"counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-03-24T08:02:53.985720","created_date":"2025-10-10T00:00:00"}