{"id":"https://openalex.org/W4413847221","doi":"https://doi.org/10.1109/tpami.2025.3604091","title":"Spherical Vision Transformers for Audio-Visual Saliency Prediction in 360$^{\\circ }$\u2218 Videos","display_name":"Spherical Vision Transformers for Audio-Visual Saliency Prediction in 360$^{\\circ }$\u2218 Videos","publication_year":2025,"publication_date":"2025-08-29","ids":{"openalex":"https://openalex.org/W4413847221","doi":"https://doi.org/10.1109/tpami.2025.3604091","pmid":"https://pubmed.ncbi.nlm.nih.gov/40880339"},"language":"en","primary_location":{"id":"doi:10.1109/tpami.2025.3604091","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2025.3604091","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5019736541","display_name":"Mert Cokelek","orcid":null},"institutions":[{"id":"https://openalex.org/I1351752","display_name":"Ko\u00e7 University","ror":"https://ror.org/00jzwgz36","country_code":"TR","type":"education","lineage":["https://openalex.org/I1351752"]}],"countries":["TR"],"is_corresponding":false,"raw_author_name":"Mert Cokelek","raw_affiliation_strings":["Department of Computer Science and Engineering, Koç University, Istanbul, Türkiye","Department of Computer Science and Engineering, Koç University, Istanbul, TR, Türkey"],"raw_orcid":"https://orcid.org/0009-0005-0577-8908","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Koç University, Istanbul, Türkiye","institution_ids":["https://openalex.org/I1351752"]},{"raw_affiliation_string":"Department of Computer Science and Engineering, Koç University, Istanbul, TR, Türkey","institution_ids":["https://openalex.org/I1351752"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119484544","display_name":"Halit Ozsoy","orcid":null},"institutions":[{"id":"https://openalex.org/I1301968116","display_name":"MEF University","ror":"https://ror.org/05jz51y94","country_code":"TR","type":"education","lineage":["https://openalex.org/I1301968116"]},{"id":"https://openalex.org/I2799978770","display_name":"X-Fab (Germany)","ror":"https://ror.org/030bh9196","country_code":"DE","type":"company","lineage":["https://openalex.org/I2799978770"]}],"countries":["DE","TR"],"is_corresponding":false,"raw_author_name":"Halit Ozsoy","raw_affiliation_strings":["Department of Psychology, Boğaziçi University, Istanbul, Türkiye","Department of Psychology, Boğaziçi University, Istanbul, TR, Türkey"],"raw_orcid":"https://orcid.org/0000-0001-8529-026X","affiliations":[{"raw_affiliation_string":"Department of Psychology, Boğaziçi University, Istanbul, Türkiye","institution_ids":["https://openalex.org/I2799978770"]},{"raw_affiliation_string":"Department of Psychology, Boğaziçi University, Istanbul, TR, Türkey","institution_ids":["https://openalex.org/I1301968116"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074611723","display_name":"Nevrez \u0130mamo\u011flu","orcid":"https://orcid.org/0000-0002-2661-599X"},"institutions":[{"id":"https://openalex.org/I73613424","display_name":"National Institute of Advanced Industrial Science and Technology","ror":"https://ror.org/01703db54","country_code":"JP","type":"government","lineage":["https://openalex.org/I73613424"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Nevrez Imamoglu","raw_affiliation_strings":["National Institute of Advanced Industrial Science and Technology (AIST), Intelligent Platforms Research Institute, Tokyo 135-0064, Japan","National Institute of Advanced Industrial Science and Technology (AIST), Intelligent Platforms Research Institute, Tokyo, Japan"],"raw_orcid":"https://orcid.org/0000-0002-2661-599X","affiliations":[{"raw_affiliation_string":"National Institute of Advanced Industrial Science and Technology (AIST), Intelligent Platforms Research Institute, Tokyo 135-0064, Japan","institution_ids":["https://openalex.org/I73613424"]},{"raw_affiliation_string":"National Institute of Advanced Industrial Science and Technology (AIST), Intelligent Platforms Research Institute, Tokyo, Japan","institution_ids":["https://openalex.org/I73613424"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003249945","display_name":"Cagri Ozcinar","orcid":"https://orcid.org/0000-0003-4915-2251"},"institutions":[{"id":"https://openalex.org/I2801810480","display_name":"Multiple Sclerosis Society","ror":"https://ror.org/043fwdk81","country_code":"GB","type":"nonprofit","lineage":["https://openalex.org/I2801810480"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Cagri Ozcinar","raw_affiliation_strings":["MSK.AI, London, U.K"],"raw_orcid":"https://orcid.org/0000-0003-4915-2251","affiliations":[{"raw_affiliation_string":"MSK.AI, London, U.K","institution_ids":["https://openalex.org/I2801810480"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022035103","display_name":"\u0130nci Ayhan","orcid":"https://orcid.org/0000-0003-0963-4299"},"institutions":[{"id":"https://openalex.org/I1301968116","display_name":"MEF University","ror":"https://ror.org/05jz51y94","country_code":"TR","type":"education","lineage":["https://openalex.org/I1301968116"]},{"id":"https://openalex.org/I2799978770","display_name":"X-Fab (Germany)","ror":"https://ror.org/030bh9196","country_code":"DE","type":"company","lineage":["https://openalex.org/I2799978770"]}],"countries":["DE","TR"],"is_corresponding":false,"raw_author_name":"Inci Ayhan","raw_affiliation_strings":["Department of Psychology, Boğaziçi University, Istanbul, Türkiye","Department of Psychology, Boğaziçi University, Istanbul, TR, Türkey"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Psychology, Boğaziçi University, Istanbul, Türkiye","institution_ids":["https://openalex.org/I2799978770"]},{"raw_affiliation_string":"Department of Psychology, Boğaziçi University, Istanbul, TR, Türkey","institution_ids":["https://openalex.org/I1301968116"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074809120","display_name":"Erkut Erdem","orcid":"https://orcid.org/0000-0002-6744-8614"},"institutions":[{"id":"https://openalex.org/I66514158","display_name":"Hacettepe University","ror":"https://ror.org/04kwvgz42","country_code":"TR","type":"education","lineage":["https://openalex.org/I66514158"]}],"countries":["TR"],"is_corresponding":false,"raw_author_name":"Erkut Erdem","raw_affiliation_strings":["Department of Computer Engineering, Hacettepe University, Ankara, Türkiye","Department of Computer Engineering, Hacettepe University, Ankara, TR, Türkey"],"raw_orcid":"https://orcid.org/0000-0002-6744-8614","affiliations":[{"raw_affiliation_string":"Department of Computer Engineering, Hacettepe University, Ankara, Türkiye","institution_ids":["https://openalex.org/I66514158"]},{"raw_affiliation_string":"Department of Computer Engineering, Hacettepe University, Ankara, TR, Türkey","institution_ids":["https://openalex.org/I66514158"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5000080119","display_name":"Aykut Erdem","orcid":"https://orcid.org/0000-0002-6280-8422"},"institutions":[{"id":"https://openalex.org/I1351752","display_name":"Ko\u00e7 University","ror":"https://ror.org/00jzwgz36","country_code":"TR","type":"education","lineage":["https://openalex.org/I1351752"]}],"countries":["TR"],"is_corresponding":false,"raw_author_name":"Aykut Erdem","raw_affiliation_strings":["Department of Computer Science and Engineering, Koç University, Istanbul, Türkiye","Department of Computer Science and Engineering, Koç University, and KUIS AI Center, Istanbul, TR, Türkey"],"raw_orcid":"https://orcid.org/0000-0002-6280-8422","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Koç University, Istanbul, Türkiye","institution_ids":["https://openalex.org/I1351752"]},{"raw_affiliation_string":"Department of Computer Science and Engineering, Koç University, and KUIS AI Center, Istanbul, TR, Türkey","institution_ids":["https://openalex.org/I1351752"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.2611,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.82206636,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":97},"biblio":{"volume":"48","issue":"1","first_page":"329","last_page":"345"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12032","display_name":"Multisensory perception and integration","score":0.9584000110626221,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12032","display_name":"Multisensory perception and integration","score":0.9584000110626221,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12496","display_name":"Color perception and design","score":0.9345999956130981,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.7404152750968933},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.7256959080696106},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6366428136825562},{"id":"https://openalex.org/keywords/audio-visual","display_name":"Audio visual","score":0.5376518964767456},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.478263258934021},{"id":"https://openalex.org/keywords/computer-graphics","display_name":"Computer graphics (images)","score":0.3675914704799652},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.36444640159606934},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.17307928204536438},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.1164972186088562}],"concepts":[{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.7404152750968933},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.7256959080696106},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6366428136825562},{"id":"https://openalex.org/C3017588708","wikidata":"https://www.wikidata.org/wiki/Q758901","display_name":"Audio visual","level":2,"score":0.5376518964767456},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.478263258934021},{"id":"https://openalex.org/C121684516","wikidata":"https://www.wikidata.org/wiki/Q7600677","display_name":"Computer graphics (images)","level":1,"score":0.3675914704799652},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.36444640159606934},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.17307928204536438},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.1164972186088562},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tpami.2025.3604091","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2025.3604091","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},{"id":"pmid:40880339","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/40880339","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on pattern analysis and machine intelligence","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":63,"referenced_works":["https://openalex.org/W1965305672","https://openalex.org/W1994202260","https://openalex.org/W2031829129","https://openalex.org/W2072500831","https://openalex.org/W2311029216","https://openalex.org/W2344823861","https://openalex.org/W2551546500","https://openalex.org/W2593116425","https://openalex.org/W2613701849","https://openalex.org/W2729969528","https://openalex.org/W2732026016","https://openalex.org/W2733888878","https://openalex.org/W2751105188","https://openalex.org/W2798665861","https://openalex.org/W2799064164","https://openalex.org/W2803485280","https://openalex.org/W2810323393","https://openalex.org/W2884414611","https://openalex.org/W2902009049","https://openalex.org/W2939731335","https://openalex.org/W2947232148","https://openalex.org/W2963155035","https://openalex.org/W2981374717","https://openalex.org/W3000351820","https://openalex.org/W3015371781","https://openalex.org/W3019533076","https://openalex.org/W3020127778","https://openalex.org/W3033059042","https://openalex.org/W3034287518","https://openalex.org/W3034304805","https://openalex.org/W3034515714","https://openalex.org/W3036714510","https://openalex.org/W3081459285","https://openalex.org/W3096895079","https://openalex.org/W3102351864","https://openalex.org/W3111243176","https://openalex.org/W3112280458","https://openalex.org/W3117908628","https://openalex.org/W3126721948","https://openalex.org/W3143329581","https://openalex.org/W3180722424","https://openalex.org/W3205475937","https://openalex.org/W3205725084","https://openalex.org/W3212456028","https://openalex.org/W4213116884","https://openalex.org/W4214564035","https://openalex.org/W4226074673","https://openalex.org/W4289387930","https://openalex.org/W4298140583","https://openalex.org/W4312607836","https://openalex.org/W4312653918","https://openalex.org/W4312724358","https://openalex.org/W4313174897","https://openalex.org/W4313476633","https://openalex.org/W4327662052","https://openalex.org/W4367279925","https://openalex.org/W4375868784","https://openalex.org/W4377997064","https://openalex.org/W4385976096","https://openalex.org/W4391488862","https://openalex.org/W4391547587","https://openalex.org/W4402715853","https://openalex.org/W4410342704"],"related_works":["https://openalex.org/W2271369634","https://openalex.org/W3147472394","https://openalex.org/W2047100085","https://openalex.org/W2350550760","https://openalex.org/W578794879","https://openalex.org/W2625296515","https://openalex.org/W3137890128","https://openalex.org/W1984634519","https://openalex.org/W4245955731","https://openalex.org/W2393726419"],"abstract_inverted_index":{"Omnidirectional":[0],"videos":[1],"(ODVs)":[2],"are":[3],"redefining":[4],"viewer":[5,166],"experiences":[6],"in":[7,61,107,164,168,181,191],"virtual":[8],"reality":[9],"(VR)":[10],"by":[11,47,65],"offering":[12],"an":[13],"unprecedented":[14],"full":[15],"field-of-view":[16],"(FOV).":[17],"This":[18],"study":[19,77],"extends":[20],"the":[21,30,36,58,66,182],"domain":[22],"of":[23,32,38,68,83,149],"saliency":[24,74,106,117,189],"prediction":[25,118,190],"to":[26,95,98,102],"360$^\\circ$\u2218":[27,72,108,169],"environments,":[28],"addressing":[29],"complexities":[31],"spherical":[33,62,128],"distortion":[34],"and":[35,133,158],"integration":[37],"spatial":[39,50,178],"audio.":[40],"Contextually,":[41],"ODVs":[42,125],"have":[43],"transformed":[44],"user":[45],"experience":[46],"adding":[48],"a":[49,80,121,147],"audio":[51,142,179],"dimension":[52],"that":[53,156,176],"aligns":[54],"sound":[55],"direction":[56],"with":[57,127],"viewer's":[59],"perspective":[60],"scenes.":[63,170],"Motivated":[64],"lack":[67],"comprehensive":[69],"datasets":[70],"for":[71,124,187],"audio-visual":[73,90,100],"prediction,":[75],"our":[76,153],"curates":[78],"YT360-EyeTracking,":[79,154],"new":[81],"dataset":[82],"81":[84],"ODVs,":[85],"each":[86],"observed":[87],"under":[88],"varying":[89],"conditions.":[91],"Our":[92,144],"goal":[93],"is":[94,185],"explore":[96],"how":[97],"utilize":[99],"cues":[101,180],"effectively":[103],"predict":[104],"visual":[105],"videos.":[109,193],"Towards":[110],"this":[111],"aim,":[112],"we":[113,174],"propose":[114],"two":[115],"novel":[116],"models:":[119],"SalViT360,":[120],"vision-transformer-based":[122],"framework":[123],"equipped":[126],"geometry-aware":[129],"spatio-temporal":[130],"attention":[131,167],"layers,":[132],"SalViT360-AV,":[134],"which":[135],"further":[136],"incorporates":[137],"transformer":[138],"adapters":[139],"conditioned":[140],"on":[141,146],"input.":[143],"results":[145],"number":[148],"benchmark":[150],"datasets,":[151],"including":[152],"demonstrate":[155],"SalViT360":[157],"SalViT360-AV":[159],"significantly":[160],"outperform":[161],"existing":[162],"methods":[163],"predicting":[165],"Interpreting":[171],"these":[172],"results,":[173],"suggest":[175],"integrating":[177],"model":[183],"architecture":[184],"crucial":[186],"accurate":[188],"omnidirectional":[192]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}