{"id":"https://openalex.org/W4406461044","doi":"https://doi.org/10.1109/bigdata62323.2024.10825630","title":"Zero-th Order Algorithm for Softmax Attention Optimization","display_name":"Zero-th Order Algorithm for Softmax Attention Optimization","publication_year":2024,"publication_date":"2024-12-15","ids":{"openalex":"https://openalex.org/W4406461044","doi":"https://doi.org/10.1109/bigdata62323.2024.10825630"},"language":"en","primary_location":{"id":"doi:10.1109/bigdata62323.2024.10825630","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata62323.2024.10825630","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Big Data (BigData)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5033655357","display_name":"Yichuan Deng","orcid":"https://orcid.org/0000-0003-1492-4370"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yichuan Deng","raw_affiliation_strings":["University of Washington,Paul G. Allen School of CSE,Seattle,WA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Washington,Paul G. Allen School of CSE,Seattle,WA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101804803","display_name":"Zhihang Li","orcid":"https://orcid.org/0000-0002-9305-7924"},"institutions":[{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhihang Li","raw_affiliation_strings":["UCSD,Department of CSE,San Diego,CA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UCSD,Department of CSE,San Diego,CA","institution_ids":["https://openalex.org/I36258959"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061960274","display_name":"Sridhar Mahadevan","orcid":null},"institutions":[{"id":"https://openalex.org/I1306409833","display_name":"Adobe Systems (United States)","ror":"https://ror.org/059tvcg64","country_code":"US","type":"company","lineage":["https://openalex.org/I1306409833"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sridhar Mahadevan","raw_affiliation_strings":["Adobe Research Adobe,San Jose,CA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Adobe Research Adobe,San Jose,CA","institution_ids":["https://openalex.org/I1306409833"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100540442","display_name":"Zhao Song","orcid":"https://orcid.org/0000-0001-5263-7044"},"institutions":[{"id":"https://openalex.org/I4210107338","display_name":"Simons Foundation","ror":"https://ror.org/01cmst727","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210107338"]},{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhao Song","raw_affiliation_strings":["UC Berkeley,Simons Institute,Berkeley,CA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UC Berkeley,Simons Institute,Berkeley,CA","institution_ids":["https://openalex.org/I4210107338","https://openalex.org/I95457486"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.3556,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.61385646,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"24","last_page":"33"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.994700014591217,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.994700014591217,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9932000041007996,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.9922999739646912,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/softmax-function","display_name":"Softmax function","score":0.8221049308776855},{"id":"https://openalex.org/keywords/zero","display_name":"Zero (linguistics)","score":0.7081677913665771},{"id":"https://openalex.org/keywords/zero-order","display_name":"Zero order","score":0.6351590156555176},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.530109167098999},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.5156139731407166},{"id":"https://openalex.org/keywords/order","display_name":"Order (exchange)","score":0.4800916016101837},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.27188974618911743},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.270466148853302},{"id":"https://openalex.org/keywords/first-order","display_name":"First order","score":0.25490182638168335},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.14704826474189758},{"id":"https://openalex.org/keywords/applied-mathematics","display_name":"Applied mathematics","score":0.13763156533241272},{"id":"https://openalex.org/keywords/philosophy","display_name":"Philosophy","score":0.05245846509933472}],"concepts":[{"id":"https://openalex.org/C188441871","wikidata":"https://www.wikidata.org/wiki/Q7554146","display_name":"Softmax function","level":3,"score":0.8221049308776855},{"id":"https://openalex.org/C2780813799","wikidata":"https://www.wikidata.org/wiki/Q3274237","display_name":"Zero (linguistics)","level":2,"score":0.7081677913665771},{"id":"https://openalex.org/C2993784581","wikidata":"https://www.wikidata.org/wiki/Q8069600","display_name":"Zero order","level":3,"score":0.6351590156555176},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.530109167098999},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.5156139731407166},{"id":"https://openalex.org/C182306322","wikidata":"https://www.wikidata.org/wiki/Q1779371","display_name":"Order (exchange)","level":2,"score":0.4800916016101837},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.27188974618911743},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.270466148853302},{"id":"https://openalex.org/C2987642246","wikidata":"https://www.wikidata.org/wiki/Q5452196","display_name":"First order","level":2,"score":0.25490182638168335},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.14704826474189758},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.13763156533241272},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.05245846509933472},{"id":"https://openalex.org/C10138342","wikidata":"https://www.wikidata.org/wiki/Q43015","display_name":"Finance","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/bigdata62323.2024.10825630","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata62323.2024.10825630","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Big Data (BigData)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":80,"referenced_works":["https://openalex.org/W826455576","https://openalex.org/W2009797711","https://openalex.org/W2072690700","https://openalex.org/W2124289529","https://openalex.org/W2128452997","https://openalex.org/W2131241448","https://openalex.org/W2149479912","https://openalex.org/W2171074980","https://openalex.org/W2171830216","https://openalex.org/W2596367596","https://openalex.org/W2746600820","https://openalex.org/W2890326782","https://openalex.org/W2896457183","https://openalex.org/W2963248893","https://openalex.org/W2963470657","https://openalex.org/W2986078788","https://openalex.org/W2986154550","https://openalex.org/W2998732476","https://openalex.org/W3041856066","https://openalex.org/W3106412272","https://openalex.org/W3136297071","https://openalex.org/W3212336101","https://openalex.org/W4224308101","https://openalex.org/W4229005866","https://openalex.org/W4287271192","https://openalex.org/W4292779060","https://openalex.org/W4295838474","https://openalex.org/W4313304716","https://openalex.org/W4319452930","https://openalex.org/W4322716742","https://openalex.org/W4323706304","https://openalex.org/W4360836968","https://openalex.org/W4361230827","https://openalex.org/W4361806691","https://openalex.org/W4362679267","https://openalex.org/W4364385332","https://openalex.org/W4366735484","https://openalex.org/W4367189889","https://openalex.org/W4367694567","https://openalex.org/W4375959325","https://openalex.org/W4378770452","https://openalex.org/W4379539416","https://openalex.org/W4380136302","https://openalex.org/W4381253585","https://openalex.org/W4382319938","https://openalex.org/W4383473983","https://openalex.org/W4385245566","https://openalex.org/W6623228368","https://openalex.org/W6678911119","https://openalex.org/W6735641298","https://openalex.org/W6754787993","https://openalex.org/W6755207826","https://openalex.org/W6759158001","https://openalex.org/W6769694023","https://openalex.org/W6770131634","https://openalex.org/W6771626834","https://openalex.org/W6778883912","https://openalex.org/W6790328413","https://openalex.org/W6791798838","https://openalex.org/W6810081322","https://openalex.org/W6811340617","https://openalex.org/W6849081016","https://openalex.org/W6849934826","https://openalex.org/W6849952668","https://openalex.org/W6850936240","https://openalex.org/W6851015762","https://openalex.org/W6851096460","https://openalex.org/W6851409614","https://openalex.org/W6851907387","https://openalex.org/W6851978508","https://openalex.org/W6851995079","https://openalex.org/W6852064026","https://openalex.org/W6852164419","https://openalex.org/W6852307388","https://openalex.org/W6853081871","https://openalex.org/W6853192989","https://openalex.org/W6853464593","https://openalex.org/W6853534439","https://openalex.org/W6853999612","https://openalex.org/W6855017244"],"related_works":["https://openalex.org/W2792321351","https://openalex.org/W2011296092","https://openalex.org/W4403194573","https://openalex.org/W2390076355","https://openalex.org/W2351616133","https://openalex.org/W2359538068","https://openalex.org/W3104605355","https://openalex.org/W2887319909","https://openalex.org/W2061019404","https://openalex.org/W4404311502"],"abstract_inverted_index":{"Large":[0],"language":[1,163],"models":[2],"(LLMs)":[3],"have":[4],"brought":[5],"about":[6],"significant":[7],"transformations":[8],"in":[9,16,70,137,158],"human":[10],"society.":[11],"Among":[12],"the":[13,18,26,48,51,60,80,88,91,96,107,129,146,153,159],"crucial":[14],"computations":[15],"LLMs,":[17,94],"softmax":[19,64],"unit":[20,65],"holds":[21],"great":[22],"importance.":[23],"Its":[24],"helps":[25],"model":[27,49],"generating":[28],"a":[29,39,67,118],"probability":[30],"distribution":[31],"on":[32,59],"potential":[33],"subsequent":[34],"words":[35],"or":[36,56],"phrases,":[37],"considering":[38],"series":[40],"of":[41,82,90,93,131,155,161],"input":[42],"words.":[43],"By":[44,144],"utilizing":[45],"this":[46,114],"distribution,":[47],"selects":[50],"most":[52],"probable":[53],"next":[54],"word":[55],"phrase,":[57],"based":[58],"assigned":[61],"probabilities.":[62],"The":[63],"assumes":[66],"vital":[68],"function":[69],"LLM":[71],"training":[72],"as":[73],"it":[74],"facilitates":[75],"learning":[76],"from":[77],"data":[78],"through":[79],"adjustment":[81],"neural":[83],"network":[84],"weights":[85],"and":[86],"biases.With":[87],"development":[89],"size":[92],"computing":[95,139],"gradient":[97,108],"becomes":[98],"expensive.":[99],"However,":[100],"Zero-th":[101,119],"Order":[102,120],"method":[103],"can":[104],"approximately":[105],"compute":[106],"with":[109],"only":[110],"forward":[111],"passes.":[112],"In":[113],"paper,":[115],"we":[116],"present":[117],"algorithm":[121],"specifically":[122],"tailored":[123],"for":[124,141],"Softmax":[125],"optimization.":[126],"We":[127],"demonstrate":[128],"convergence":[130],"our":[132,149],"algorithm,":[133],"highlighting":[134],"its":[135],"effectiveness":[136],"efficiently":[138],"gradients":[140],"large-scale":[142],"LLMs.":[143],"leveraging":[145],"Zeroth-Order":[147],"method,":[148],"work":[150],"contributes":[151],"to":[152],"advancement":[154],"optimization":[156],"techniques":[157],"context":[160],"complex":[162],"models.":[164]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}