{"id":"https://openalex.org/W4406959633","doi":"https://doi.org/10.48550/arxiv.2501.16753","title":"Overcoming Semantic Dilution in Transformer-Based Next Frame Prediction","display_name":"Overcoming Semantic Dilution in Transformer-Based Next Frame Prediction","publication_year":2025,"publication_date":"2025-01-28","ids":{"openalex":"https://openalex.org/W4406959633","doi":"https://doi.org/10.48550/arxiv.2501.16753"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2501.16753","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2501.16753","pdf_url":"https://arxiv.org/pdf/2501.16753","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2501.16753","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5036572807","display_name":"Hy Nguyen","orcid":"https://orcid.org/0009-0008-7092-3106","observed_orcids":["https://orcid.org/0009-0008-7092-3106"]},"institutions":[{"id":"https://openalex.org/I149704539","display_name":"Deakin University","ror":"https://ror.org/02czsnj07","country_code":"AU","type":"education","lineage":["https://openalex.org/I149704539"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Nguyen, Hy","raw_affiliation_strings":["Applied Artificial Intelligence Institute , Deakin University Burwood , Victoria , Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Applied Artificial Intelligence Institute , Deakin University Burwood , Victoria , Australia","institution_ids":["https://openalex.org/I149704539"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070118830","display_name":"Srikanth Thudumu","orcid":"https://orcid.org/0000-0002-7848-9008","observed_orcids":["https://orcid.org/0000-0002-7848-9008"]},"institutions":[{"id":"https://openalex.org/I149704539","display_name":"Deakin University","ror":"https://ror.org/02czsnj07","country_code":"AU","type":"education","lineage":["https://openalex.org/I149704539"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Thudumu, Srikanth","raw_affiliation_strings":["Applied Artificial Intelligence Institute , Deakin University Burwood , Victoria , Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Applied Artificial Intelligence Institute , Deakin University Burwood , Victoria , Australia","institution_ids":["https://openalex.org/I149704539"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048022541","display_name":"Hung Du","orcid":"https://orcid.org/0000-0003-1415-5786","observed_orcids":["https://orcid.org/0000-0003-1415-5786"]},"institutions":[{"id":"https://openalex.org/I149704539","display_name":"Deakin University","ror":"https://ror.org/02czsnj07","country_code":"AU","type":"education","lineage":["https://openalex.org/I149704539"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Du, Hung","raw_affiliation_strings":["Applied Artificial Intelligence Institute , Deakin University Burwood , Victoria , Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Applied Artificial Intelligence Institute , Deakin University Burwood , Victoria , Australia","institution_ids":["https://openalex.org/I149704539"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030486012","display_name":"Rajesh Vasa","orcid":"https://orcid.org/0000-0003-4805-1467","observed_orcids":["https://orcid.org/0000-0003-4805-1467","https://orcid.org/0000-0001-7230-6541"]},"institutions":[{"id":"https://openalex.org/I149704539","display_name":"Deakin University","ror":"https://ror.org/02czsnj07","country_code":"AU","type":"education","lineage":["https://openalex.org/I149704539"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Vasa, Rajesh","raw_affiliation_strings":["Applied Artificial Intelligence Institute , Deakin University Burwood , Victoria , Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Applied Artificial Intelligence Institute , Deakin University Burwood , Victoria , Australia","institution_ids":["https://openalex.org/I149704539"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5044290815","display_name":"Kon Mouzakis","orcid":"https://orcid.org/0000-0003-4447-5166","observed_orcids":["https://orcid.org/0000-0003-4447-5166"]},"institutions":[{"id":"https://openalex.org/I149704539","display_name":"Deakin University","ror":"https://ror.org/02czsnj07","country_code":"AU","type":"education","lineage":["https://openalex.org/I149704539"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Mouzakis, Kon","raw_affiliation_strings":["Applied Artificial Intelligence Institute , Deakin University Burwood , Victoria , Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Applied Artificial Intelligence Institute , Deakin University Burwood , Victoria , Australia","institution_ids":["https://openalex.org/I149704539"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I149704539"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.9312000274658203,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.9312000274658203,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.01759999990463257,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.014800000004470348,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/next-frame-prediction","display_name":"next-frame prediction","score":0.9129999876022339},{"id":"https://openalex.org/keywords/video-prediction","display_name":"video prediction","score":0.8309999704360962},{"id":"https://openalex.org/keywords/multi-head-self-attention","display_name":"multi-head self-attention","score":0.9900000095367432},{"id":"https://openalex.org/keywords/transformer","display_name":"transformer models","score":0.45399999618530273},{"id":"https://openalex.org/keywords/autonomous-driving","display_name":"autonomous driving","score":0.5210000276565552}],"concepts":[{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6773133277893066},{"id":"https://openalex.org/C96618451","wikidata":"https://www.wikidata.org/wiki/Q3028008","display_name":"Dilution","level":2,"score":0.6116995811462402},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5435656309127808},{"id":"https://openalex.org/C126042441","wikidata":"https://www.wikidata.org/wiki/Q1324888","display_name":"Frame (networking)","level":2,"score":0.4462927579879761},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3450791835784912},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.32890549302101135},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.14926838874816895},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.1175805926322937},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.1035037636756897},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.08770549297332764},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.07283475995063782},{"id":"https://openalex.org/C97355855","wikidata":"https://www.wikidata.org/wiki/Q11473","display_name":"Thermodynamics","level":1,"score":0.04909345507621765}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:arXiv.org:2501.16753","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2501.16753","pdf_url":"https://arxiv.org/pdf/2501.16753","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:aisel.aisnet.org:amcis2025-1604","doi":null,"is_oa":true,"landing_page_url":"https://aisel.aisnet.org/amcis2025/urbanmob/urbanmob/3","pdf_url":"https://aisel.aisnet.org/cgi/viewcontent.cgi?article=1604&context=amcis2025","source":{"id":"https://openalex.org/S30879505","display_name":"Journal of the Association for Information Systems","issn_l":"1536-9323","issn":["1536-9323"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["abdc-a-star","anvur-scientific","cwts-core","dongbi-a","fnege-1","jufo-3","norway-2","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310321080","host_organization_name":"Association for Information Systems","host_organization_lineage":["https://openalex.org/P4310321080"],"host_organization_lineage_names":["Association for Information Systems"],"type":"journal"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"AMCIS 2025 Proceedings","raw_type":"text"},{"id":"doi:10.48550/arxiv.2501.16753","doi":"https://doi.org/10.48550/arxiv.2501.16753","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2501.16753","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2501.16753","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2501.16753","pdf_url":"https://arxiv.org/pdf/2501.16753","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4406959633.pdf","grobid_xml":"https://content.openalex.org/works/W4406959633.grobid-xml"},"referenced_works_count":3,"referenced_works":["https://openalex.org/W2248556341","https://openalex.org/W2908510526","https://openalex.org/W3094502228"],"related_works":["https://openalex.org/W4391456595","https://openalex.org/W2919455813","https://openalex.org/W4312305807","https://openalex.org/W4408100792","https://openalex.org/W4392207693","https://openalex.org/W2583901669","https://openalex.org/W7148211322","https://openalex.org/W7223367426","https://openalex.org/W2901525235","https://openalex.org/W3016475064"],"abstract_inverted_index":{"Next-frame":[0],"prediction":[1,22,59],"in":[2,20,24,45,53,105,110,173,186],"videos":[3],"is":[4,82],"crucial":[5],"for":[6,42],"applications":[7],"such":[8],"as":[9],"autonomous":[10],"driving,":[11],"object":[12],"tracking,":[13],"and":[14,27,31,155],"motion":[15],"prediction.":[16,176],"The":[17,38,65],"primary":[18],"challenge":[19],"next-frame":[21,58,175],"lies":[23],"effectively":[25,169],"capturing":[26],"processing":[28],"both":[29],"spatial":[30],"temporal":[32],"information":[33],"from":[34],"previous":[35],"video":[36],"sequences.":[37],"transformer":[39],"architecture,":[40,167],"known":[41],"its":[43],"prowess":[44],"handling":[46],"sequence":[47],"data,":[48],"has":[49],"made":[50],"remarkable":[51],"progress":[52],"this":[54,147],"domain.":[55],"However,":[56],"transformer-based":[57,174,209],"models":[60,117],"face":[61],"notable":[62],"issues:":[63],"(a)":[64],"multi-head":[66],"self-attention":[67],"(MHSA)":[68],"mechanism":[69],"requires":[70],"the":[71,83,94,100,103,106,119,122,127,131,136,139,143,152,156,187,191,197,207],"input":[72],"embedding":[73,104],"to":[74,206],"be":[75],"split":[76],"into":[77],"$N$":[78,81],"chunks,":[79],"where":[80],"number":[84],"of":[85,93,102,121,138],"heads.":[86],"Each":[87],"segment":[88],"captures":[89],"only":[90],"a":[91,111,149,161,180],"fraction":[92],"original":[95,208],"embeddings":[96,120,145],"information,":[97],"which":[98,168],"distorts":[99],"representation":[101],"latent":[107,188],"space,":[108,189],"resulting":[109],"semantic":[112,171],"dilution":[113,172],"problem;":[114],"(b)":[115],"These":[116],"predict":[118],"next":[123],"frames":[124,128],"rather":[125],"than":[126],"themselves,":[129],"but":[130],"loss":[132,181],"function":[133,182],"based":[134],"on":[135],"errors":[137],"reconstructed":[140],"frames,":[141],"not":[142],"predicted":[144],"--":[146],"creates":[148],"discrepancy":[150],"between":[151],"training":[153,192],"objective":[154,193],"model":[157,198],"output.":[158,199],"We":[159],"propose":[160],"Semantic":[162],"Concentration":[163],"Multi-Head":[164],"Self-Attention":[165],"(SCMHSA)":[166],"mitigates":[170],"Additionally,":[177],"we":[178],"introduce":[179],"that":[183],"optimizes":[184],"SCMHSA":[185],"aligning":[190],"more":[194],"closely":[195],"with":[196],"Our":[200],"method":[201],"demonstrates":[202],"superior":[203],"performance":[204],"compared":[205],"predictors.":[210]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2025-01-30T00:00:00"}