{"id":"https://openalex.org/W7161990895","doi":"https://doi.org/10.48550/arxiv.2605.20936","title":"DASH: Fast Differentiable Architecture Search for Hybrid Attention in Minutes on a Single GPU","display_name":"DASH: Fast Differentiable Architecture Search for Hybrid Attention in Minutes on a Single GPU","publication_year":2026,"publication_date":"2026-05-20","ids":{"openalex":"https://openalex.org/W7161990895","doi":"https://doi.org/10.48550/arxiv.2605.20936"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.20936","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.20936","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.20936","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103236436","display_name":"Weizhe Chen","orcid":"https://orcid.org/0000-0001-9068-4247"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Weizhe","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136626719","display_name":"Miao Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Miao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047142881","display_name":"Junpeng Jiang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jiang, Junpeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136639852","display_name":"Yaping Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Yaping","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136635356","display_name":"Weili Guan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guan, Weili","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136628541","display_name":"Liqiang Nie","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nie, Liqiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.7182999849319458,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.7182999849319458,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.04520000144839287,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.01549999974668026,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/architecture","display_name":"Architecture","score":0.6376000046730042},{"id":"https://openalex.org/keywords/differentiable-function","display_name":"Differentiable function","score":0.5805000066757202},{"id":"https://openalex.org/keywords/hybrid-system","display_name":"Hybrid system","score":0.5228000283241272},{"id":"https://openalex.org/keywords/operator","display_name":"Operator (biology)","score":0.48919999599456787},{"id":"https://openalex.org/keywords/suite","display_name":"Suite","score":0.4311000108718872},{"id":"https://openalex.org/keywords/search-algorithm","display_name":"Search algorithm","score":0.40939998626708984},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.30790001153945923}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7893000245094299},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.6376000046730042},{"id":"https://openalex.org/C202615002","wikidata":"https://www.wikidata.org/wiki/Q783507","display_name":"Differentiable function","level":2,"score":0.5805000066757202},{"id":"https://openalex.org/C50897621","wikidata":"https://www.wikidata.org/wiki/Q2665508","display_name":"Hybrid system","level":2,"score":0.5228000283241272},{"id":"https://openalex.org/C17020691","wikidata":"https://www.wikidata.org/wiki/Q139677","display_name":"Operator (biology)","level":5,"score":0.48919999599456787},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4505999982357025},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.4311000108718872},{"id":"https://openalex.org/C125583679","wikidata":"https://www.wikidata.org/wiki/Q755673","display_name":"Search algorithm","level":2,"score":0.40939998626708984},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.3709999918937683},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.334199994802475},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.30790001153945923},{"id":"https://openalex.org/C98025372","wikidata":"https://www.wikidata.org/wiki/Q477538","display_name":"Systems architecture","level":3,"score":0.30720001459121704},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3052000105381012},{"id":"https://openalex.org/C147297375","wikidata":"https://www.wikidata.org/wiki/Q6674930","display_name":"Look-ahead","level":2,"score":0.2985000014305115},{"id":"https://openalex.org/C21782646","wikidata":"https://www.wikidata.org/wiki/Q841666","display_name":"Search cost","level":2,"score":0.2865000069141388},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.27230000495910645},{"id":"https://openalex.org/C19889080","wikidata":"https://www.wikidata.org/wiki/Q2835852","display_name":"Beam search","level":3,"score":0.2705000042915344},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.25780001282691956},{"id":"https://openalex.org/C126042441","wikidata":"https://www.wikidata.org/wiki/Q1324888","display_name":"Frame (networking)","level":2,"score":0.25519999861717224},{"id":"https://openalex.org/C175309249","wikidata":"https://www.wikidata.org/wiki/Q725864","display_name":"Pipeline transport","level":2,"score":0.25440001487731934}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.20936","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.20936","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.20936","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.20936","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Hybrid":[0],"attention":[1,88,95,134,205],"architectures":[2,206],"are":[3],"becoming":[4],"an":[5],"increasingly":[6],"important":[7],"paradigm":[8],"for":[9,37,74,86,218],"improving":[10],"LLM":[11],"inference":[12],"efficiency":[13],"while":[14,157],"preserving":[15],"model":[16,112],"quality,":[17],"making":[18,64],"hybrid":[19,52,75,87,133,145,204,219],"architecture":[20,53,76,89,100,220],"design":[21,135],"a":[22,81,127,181,215],"central":[23],"problem.":[24],"Existing":[25],"designs":[26],"often":[27],"rely":[28],"on":[29,160,180],"manual":[30],"empirical":[31],"rules":[32],"or":[33],"proxy-based":[34],"selector":[35],"signals":[36],"layer-wise":[38,94],"operator":[39,96,114],"allocation.":[40],"Recent":[41],"NAS-style":[42],"systems":[43],"such":[44,65],"as":[45,71],"Jet-Nemotron":[46,155],"demonstrate":[47],"the":[48,192],"promise":[49],"of":[50,130,191],"automated":[51],"search.":[54],"However,":[55],"Jet-Nemotron's":[56],"PostNAS":[57,193],"search":[58,66,84,110,120,141,169,194],"stages":[59],"alone":[60],"use":[61,70],"200B":[62],"tokens,":[63],"pipelines":[67],"difficult":[68],"to":[69,117,188],"routine":[72],"methods":[73],"design.":[77,221],"We":[78],"introduce":[79],"DASH,":[80],"fast":[82],"differentiable":[83,140,212],"framework":[85],"design,":[90],"which":[91],"relaxes":[92],"discrete":[93],"placement":[97],"into":[98],"continuous":[99],"logits,":[101],"prepares":[102],"reusable":[103],"teacher-aligned":[104],"linear":[105],"candidates,":[106],"and":[107,113,163,175],"performs":[108],"architecture-only":[109],"with":[111],"weights":[115],"frozen":[116],"significantly":[118],"enhance":[119],"efficiency.":[121],"On":[122],"Qwen2.5-3B-Instruct,":[123],"DASH":[124,148,168],"consistently":[125],"outperforms":[126],"comprehensive":[128],"suite":[129],"existing":[131],"selector-style":[132],"baselines,":[136],"showing":[137],"that":[138,202],"direct":[139],"can":[142,207],"discover":[143],"stronger":[144,150],"architectures.":[146],"Moreover,":[147],"achieves":[149],"RULER":[151],"performance":[152],"than":[153],"released":[154],"models":[156],"remaining":[158],"competitive":[159],"overlapping":[161],"short-context":[162],"general":[164],"benchmarks.":[165],"Notably,":[166],"each":[167],"run":[170],"uses":[171],"only":[172],"12.3M":[173],"tokens":[174,195],"takes":[176],"about":[177],"20":[178],"minutes":[179],"single":[182],"RTX":[183],"Pro":[184],"6000":[185],"GPU,":[186],"corresponding":[187],"merely":[189],"0.006%":[190],"reported":[196],"by":[197],"Jet-Nemotron.":[198],"These":[199],"results":[200],"suggest":[201],"high-quality":[203],"be":[208],"obtained":[209],"through":[210],"minutes-level":[211],"search,":[213],"providing":[214],"promising":[216],"direction":[217]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-22T00:00:00"}