{"id":"https://openalex.org/W7128367917","doi":"https://doi.org/10.48550/arxiv.2602.06283","title":"SOCKET: SOft Collison Kernel EsTimator for Sparse Attention","display_name":"SOCKET: SOft Collison Kernel EsTimator for Sparse Attention","publication_year":2026,"publication_date":"2026-02-06","ids":{"openalex":"https://openalex.org/W7128367917","doi":"https://doi.org/10.48550/arxiv.2602.06283"},"language":null,"primary_location":{"id":"pmh:doi:10.48550/arxiv.2602.06283","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":null,"any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5125405010","display_name":"Sahil Joshi","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Joshi, Sahil","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020686363","display_name":"Agniva Chowdhury","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chowdhury, Agniva","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125394484","display_name":"Wyatt Bellinger","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bellinger, Wyatt","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120555334","display_name":"Amar Kanakamedala","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kanakamedala, Amar","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125429352","display_name":"Ekam Singh","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Singh, Ekam","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125378550","display_name":"Hoang Anh Duy Le","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Le, Hoang Anh Duy","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012744187","display_name":"Aditya Desai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Desai, Aditya","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5125388385","display_name":"Anshumali Shrivastava","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shrivastava, Anshumali","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5125405010"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.16449999809265137,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.16449999809265137,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.14300000667572021,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.10019999742507935,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/kernel","display_name":"Kernel (algebra)","score":0.6208999752998352},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5986999869346619},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.48170000314712524},{"id":"https://openalex.org/keywords/estimator","display_name":"Estimator","score":0.4772000014781952},{"id":"https://openalex.org/keywords/hash-function","display_name":"Hash function","score":0.4708999991416931},{"id":"https://openalex.org/keywords/collision","display_name":"Collision","score":0.4564000070095062},{"id":"https://openalex.org/keywords/selection","display_name":"Selection (genetic algorithm)","score":0.45089998841285706},{"id":"https://openalex.org/keywords/hash-table","display_name":"Hash table","score":0.4052000045776367},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.3747999966144562}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.753600001335144},{"id":"https://openalex.org/C74193536","wikidata":"https://www.wikidata.org/wiki/Q574844","display_name":"Kernel (algebra)","level":2,"score":0.6208999752998352},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5986999869346619},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.48170000314712524},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.47909998893737793},{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.4772000014781952},{"id":"https://openalex.org/C99138194","wikidata":"https://www.wikidata.org/wiki/Q183427","display_name":"Hash function","level":2,"score":0.4708999991416931},{"id":"https://openalex.org/C121704057","wikidata":"https://www.wikidata.org/wiki/Q352070","display_name":"Collision","level":2,"score":0.4564000070095062},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.45089998841285706},{"id":"https://openalex.org/C67388219","wikidata":"https://www.wikidata.org/wiki/Q207440","display_name":"Hash table","level":3,"score":0.4052000045776367},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.3747999966144562},{"id":"https://openalex.org/C37616216","wikidata":"https://www.wikidata.org/wiki/Q3218363","display_name":"Lasso (programming language)","level":2,"score":0.3610999882221222},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.35760000348091125},{"id":"https://openalex.org/C173801870","wikidata":"https://www.wikidata.org/wiki/Q201413","display_name":"Heuristic","level":2,"score":0.35199999809265137},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.34119999408721924},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.32670000195503235},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.32659998536109924},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.3222000002861023},{"id":"https://openalex.org/C122280245","wikidata":"https://www.wikidata.org/wiki/Q620622","display_name":"Kernel method","level":3,"score":0.31119999289512634},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.30469998717308044},{"id":"https://openalex.org/C159877910","wikidata":"https://www.wikidata.org/wiki/Q2202883","display_name":"Autoregressive model","level":2,"score":0.2994000017642975},{"id":"https://openalex.org/C70437156","wikidata":"https://www.wikidata.org/wiki/Q7228652","display_name":"Pooling","level":2,"score":0.2971000075340271},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.29030001163482666},{"id":"https://openalex.org/C93959086","wikidata":"https://www.wikidata.org/wiki/Q6888345","display_name":"Model selection","level":2,"score":0.28139999508857727},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.2791000008583069},{"id":"https://openalex.org/C55166926","wikidata":"https://www.wikidata.org/wiki/Q2892946","display_name":"Oracle","level":2,"score":0.2784999907016754},{"id":"https://openalex.org/C165435473","wikidata":"https://www.wikidata.org/wiki/Q1509884","display_name":"Padding","level":2,"score":0.26330000162124634},{"id":"https://openalex.org/C2777210771","wikidata":"https://www.wikidata.org/wiki/Q4927124","display_name":"Block (permutation group theory)","level":2,"score":0.25920000672340393},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.25099998712539673},{"id":"https://openalex.org/C311688","wikidata":"https://www.wikidata.org/wiki/Q2393193","display_name":"Time complexity","level":2,"score":0.25}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:doi:10.48550/arxiv.2602.06283","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"doi:10.48550/arxiv.2602.06283","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.06283","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:doi:10.48550/arxiv.2602.06283","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Exploiting":[0],"sparsity":[1],"during":[2],"long-context":[3,156,195],"inference":[4,47],"is":[5,78,87,198],"central":[6],"to":[7,28,123,183],"scaling":[8],"large":[9],"language":[10],"models,":[11],"as":[12,54],"attention":[13,21,152],"dominates":[14],"the":[15,105,111],"cost":[16,24],"of":[17,31,43,107,161],"autoregressive":[18],"decoding.":[19],"Sparse":[20],"reduces":[22],"this":[23,135],"by":[25],"restricting":[26],"computation":[27],"a":[29,55,61,120,124,164,172],"subset":[30],"tokens,":[32],"but":[33],"its":[34],"effectiveness":[35],"depends":[36],"critically":[37],"on":[38],"efficient":[39,139],"scoring":[40,129,169],"and":[41,58,86,126,146,171],"selection":[42,141],"relevant":[44],"tokens":[45],"at":[46,200],"time.":[48],"We":[49],"revisit":[50],"Locality-Sensitive":[51],"Hashing":[52],"(LSH)":[53],"sparsification":[56],"primitive":[57],"introduce":[59],"SOCKET,":[60],"SOft":[62],"Collision":[63],"Kernel":[64],"EsTimator":[65],"that":[66,79],"replaces":[67],"hard":[68,80],"bucket":[69],"matches":[70,147],"with":[71],"probabilistic,":[72],"similarity-aware":[73],"aggregation.":[74],"Our":[75],"key":[76],"insight":[77],"LSH":[81,96,118],"produces":[82],"discrete":[83],"collision":[84,99],"signals":[85],"therefore":[88],"poorly":[89],"suited":[90],"for":[91,131,168,177,194],"ranking.":[92],"In":[93],"contrast,":[94],"soft":[95],"aggregates":[97],"graded":[98],"evidence":[100],"across":[101,154],"hash":[102],"tables,":[103],"preserving":[104],"stability":[106],"relative":[108],"ordering":[109],"among":[110],"true":[112],"top-$k$":[113],"tokens.":[114],"This":[115],"transformation":[116],"elevates":[117],"from":[119],"candidate-generation":[121],"heuristic":[122],"principled":[125],"mathematically":[127],"grounded":[128],"kernel":[130,167],"sparse":[132,151,178],"attention.":[133],"Leveraging":[134],"property,":[136],"SOCKET":[137,180],"enables":[138],"token":[140],"without":[142],"ad-hoc":[143],"voting":[144],"mechanism,":[145],"or":[148],"surpasses":[149],"established":[150],"baselines":[153],"multiple":[155],"benchmarks":[157],"using":[158],"diverse":[159],"set":[160],"models.":[162],"With":[163],"custom":[165],"CUDA":[166],"keys":[170],"Flash":[173],"Decode":[174],"Triton":[175],"backend":[176],"attention,":[179],"achieves":[181],"up":[182],"1.5$\\times$":[184],"higher":[185],"throughput":[186],"than":[187],"FlashAttention,":[188],"making":[189],"it":[190],"an":[191],"effective":[192],"tool":[193],"inference.":[196],"Code":[197],"open-sourced":[199],"https://github.com/amarka8/SOCKET.":[201]},"counts_by_year":[],"updated_date":"2026-04-04T16:13:02.066488","created_date":"2026-02-10T00:00:00"}