{"id":"https://openalex.org/W4387968070","doi":"https://doi.org/10.1145/3581783.3612379","title":"SpaceCLIP: A Vision-Language Pretraining Framework With Spatial Reconstruction On Text","display_name":"SpaceCLIP: A Vision-Language Pretraining Framework With Spatial Reconstruction On Text","publication_year":2023,"publication_date":"2023-10-26","ids":{"openalex":"https://openalex.org/W4387968070","doi":"https://doi.org/10.1145/3581783.3612379"},"language":"en","primary_location":{"id":"doi:10.1145/3581783.3612379","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3581783.3612379","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3581783.3612379","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM International Conference on Multimedia","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3581783.3612379","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5066978737","display_name":"Bo Zou","orcid":"https://orcid.org/0000-0003-4047-1213"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bo Zou","raw_affiliation_strings":["Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-4047-1213","affiliations":[{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101667989","display_name":"Yang Chao","orcid":"https://orcid.org/0000-0002-2697-5661"},"institutions":[{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chao Yang","raw_affiliation_strings":["Shanghai AI Laboratory, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0002-2697-5661","affiliations":[{"raw_affiliation_string":"Shanghai AI Laboratory, Shanghai, China","institution_ids":["https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028963029","display_name":"Chengbin Quan","orcid":"https://orcid.org/0009-0007-5450-6567"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chengbin Quan","raw_affiliation_strings":["Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0007-5450-6567","affiliations":[{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101491459","display_name":"Youjian Zhao","orcid":"https://orcid.org/0000-0001-9841-1796"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Youjian Zhao","raw_affiliation_strings":["Tsinghua University & Zhongguancun Laboratory, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-9841-1796","affiliations":[{"raw_affiliation_string":"Tsinghua University & Zhongguancun Laboratory, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"519","last_page":"528"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.9937999844551086,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8378031253814697},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.697050154209137},{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.6666540503501892},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.6259573698043823},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6045806407928467},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6042518615722656},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5676738023757935},{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.4826005697250366},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4483380615711212},{"id":"https://openalex.org/keywords/natural","display_name":"Natural (archaeology)","score":0.4250178337097168}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8378031253814697},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.697050154209137},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.6666540503501892},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.6259573698043823},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6045806407928467},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6042518615722656},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5676738023757935},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.4826005697250366},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4483380615711212},{"id":"https://openalex.org/C2776608160","wikidata":"https://www.wikidata.org/wiki/Q4785462","display_name":"Natural (archaeology)","level":2,"score":0.4250178337097168},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3581783.3612379","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3581783.3612379","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3581783.3612379","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM International Conference on Multimedia","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3581783.3612379","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3581783.3612379","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3581783.3612379","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM International Conference on Multimedia","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.7799999713897705,"display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4387968070.pdf","grobid_xml":"https://content.openalex.org/works/W4387968070.grobid-xml"},"referenced_works_count":43,"referenced_works":["https://openalex.org/W2117539524","https://openalex.org/W2277195237","https://openalex.org/W2425121537","https://openalex.org/W2565656701","https://openalex.org/W2606982687","https://openalex.org/W2760103357","https://openalex.org/W2765716052","https://openalex.org/W2799067027","https://openalex.org/W2883429621","https://openalex.org/W2885775891","https://openalex.org/W2886641317","https://openalex.org/W2914699769","https://openalex.org/W2954199749","https://openalex.org/W2962949233","https://openalex.org/W2962964995","https://openalex.org/W2963351113","https://openalex.org/W2963916161","https://openalex.org/W2964089981","https://openalex.org/W2971680695","https://openalex.org/W2972073579","https://openalex.org/W2984008963","https://openalex.org/W2990503944","https://openalex.org/W2997429269","https://openalex.org/W3009803092","https://openalex.org/W3034730770","https://openalex.org/W3035265375","https://openalex.org/W3035524453","https://openalex.org/W3035635319","https://openalex.org/W3126721948","https://openalex.org/W3168640669","https://openalex.org/W3171007011","https://openalex.org/W3176398504","https://openalex.org/W3204588463","https://openalex.org/W3204670646","https://openalex.org/W3205786327","https://openalex.org/W3207758636","https://openalex.org/W3209586588","https://openalex.org/W3213562172","https://openalex.org/W3217102353","https://openalex.org/W4226289673","https://openalex.org/W4313186260","https://openalex.org/W6600376255","https://openalex.org/W6624914812"],"related_works":["https://openalex.org/W4288267738","https://openalex.org/W2964413124","https://openalex.org/W4388937922","https://openalex.org/W3157284875","https://openalex.org/W2259406085","https://openalex.org/W2099715052","https://openalex.org/W3113264705","https://openalex.org/W2147241511","https://openalex.org/W4226247999","https://openalex.org/W4213176082"],"abstract_inverted_index":{"The":[0],"tremendous":[1],"progress":[2],"of":[3,28,57],"vision-to-language":[4],"retrieval":[5],"over":[6],"these":[7],"years":[8],"is":[9,45],"fueled":[10],"by":[11],"contrastive":[12,20,75],"vision-language":[13],"pretraining":[14,106],"(VLP),":[15],"such":[16],"as":[17],"CLIP.":[18],"Although,":[19],"methods":[21,126],"do":[22],"not":[23],"exhibit":[24],"the":[25,48,55,96,111],"same":[26,97],"level":[27],"performance":[29,116,128],"on":[30,81,117,136,147,158],"other":[31,125],"downstream":[32,118],"tasks":[33],"(e.g.,":[34],"video":[35],"question":[36],"answering":[37],"and":[38,52,72,104,114,138,149,153],"natural":[39,160],"language":[40,161],"grounding).":[41],"One":[42],"possible":[43],"reason":[44],"they":[46],"ignore":[47],"misalignment":[49],"between":[50],"vision":[51],"language,":[53],"especially":[54],"absence":[56],"spatial":[58,79,98],"information":[59],"in":[60],"language.":[61],"To":[62],"mitigate":[63],"this":[64],"issue,":[65],"We":[66],"start":[67],"from":[68,131],"a":[69,74,87,105],"new":[70],"perspective":[71],"propose":[73],"VLP":[76],"framework":[77],"with":[78,100,127],"reconstruction":[80,89],"text":[82,93],"(SpaceCLIP).":[83],"Specifically,":[84],"we":[85,121],"introduce":[86],"unique":[88],"method":[90],"to":[91,109,134,145,156],"assign":[92],"representations":[94],"into":[95],"structure":[99],"images":[101],"or":[102],"videos":[103],"objective,":[107],"SpatialNCE,":[108],"reduce":[110],"computational":[112],"overhead":[113],"ensure":[115],"tasks.":[119],"Empirically,":[120],"show":[122],"SpaceCLIP":[123],"outperforms":[124],"gains":[129],"ranging":[130],"2.1%":[132],"up":[133,144,155],"9.0%":[135],"MSRVTT":[137,150],"EgoCLIP":[139],"multiple-choice":[140],"questions":[141],"answering,":[142],"2.5%":[143],"11.0%":[146],"EPIC-KITCHENS-100":[148],"multi-instance":[151],"retrieval,":[152],"0.31%":[154],"7.2%":[157],"Ego4D":[159],"query":[162],"benchmark.":[163]},"counts_by_year":[{"year":2024,"cited_by_count":1}],"updated_date":"2026-07-22T07:51:19.307946","created_date":"2025-10-10T00:00:00"}