{"id":"https://openalex.org/W2049074570","doi":"https://doi.org/10.1109/asru.2013.6707747","title":"Accelerating Hessian-free optimization for Deep Neural Networks by implicit preconditioning and sampling","display_name":"Accelerating Hessian-free optimization for Deep Neural Networks by implicit preconditioning and sampling","publication_year":2013,"publication_date":"2013-12-01","ids":{"openalex":"https://openalex.org/W2049074570","doi":"https://doi.org/10.1109/asru.2013.6707747","mag":"2049074570"},"language":"en","primary_location":{"id":"doi:10.1109/asru.2013.6707747","is_oa":false,"landing_page_url":"https://doi.org/10.1109/asru.2013.6707747","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2013 IEEE Workshop on Automatic Speech Recognition and Understanding","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/1309.1508","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5070513394","display_name":"Tara N. Sainath","orcid":"https://orcid.org/0000-0002-4126-6556"},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tara N. Sainath","raw_affiliation_strings":["IBM T. J. Watson Research Center, Yorktown Heights, NY, U.S.A","IBM T. J. Watson Research Center, Yorktown Heights , NY, USA#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM T. J. Watson Research Center, Yorktown Heights, NY, U.S.A","institution_ids":[]},{"raw_affiliation_string":"IBM T. J. Watson Research Center, Yorktown Heights , NY, USA#TAB#","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089022499","display_name":"Lior Horesh","orcid":"https://orcid.org/0000-0001-6350-0238"},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lior Horesh","raw_affiliation_strings":["IBM T. J. Watson Research Center, Yorktown Heights, NY, U.S.A","IBM T. J. Watson Research Center, Yorktown Heights , NY, USA#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM T. J. Watson Research Center, Yorktown Heights, NY, U.S.A","institution_ids":[]},{"raw_affiliation_string":"IBM T. J. Watson Research Center, Yorktown Heights , NY, USA#TAB#","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003725957","display_name":"Brian Kingsbury","orcid":"https://orcid.org/0000-0002-1343-6837"},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Brian Kingsbury","raw_affiliation_strings":["IBM T. J. Watson Research Center, Yorktown Heights, NY, U.S.A","IBM T. J. Watson Research Center, Yorktown Heights , NY, USA#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM T. J. Watson Research Center, Yorktown Heights, NY, U.S.A","institution_ids":[]},{"raw_affiliation_string":"IBM T. J. Watson Research Center, Yorktown Heights , NY, USA#TAB#","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091863515","display_name":"Aleksandr Y. Aravkin","orcid":"https://orcid.org/0000-0002-1875-1801"},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aleksandr Y. Aravkin","raw_affiliation_strings":["IBM T. J. Watson Research Center, Yorktown Heights, NY, U.S.A","IBM T. J. Watson Research Center, Yorktown Heights , NY, USA#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM T. J. Watson Research Center, Yorktown Heights, NY, U.S.A","institution_ids":[]},{"raw_affiliation_string":"IBM T. J. Watson Research Center, Yorktown Heights , NY, USA#TAB#","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5071715737","display_name":"Bhuvana Ramabhadran","orcid":null},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bhuvana Ramabhadran","raw_affiliation_strings":["IBM T. J. Watson Research Center, Yorktown Heights, NY, U.S.A","IBM T. J. Watson Research Center, Yorktown Heights , NY, USA#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM T. J. Watson Research Center, Yorktown Heights, NY, U.S.A","institution_ids":[]},{"raw_affiliation_string":"IBM T. J. Watson Research Center, Yorktown Heights , NY, USA#TAB#","institution_ids":["https://openalex.org/I1341412227"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I1341412227"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10792","display_name":"Matrix Theory and Algorithms","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10792","display_name":"Matrix Theory and Algorithms","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12303","display_name":"Tensor decomposition and applications","score":0.9979000091552734,"subfield":{"id":"https://openalex.org/subfields/2605","display_name":"Computational Mathematics"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10500","display_name":"Sparse and Compressive Sensing Techniques","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/2206","display_name":"Computational Mechanics"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/hessian-matrix","display_name":"Hessian matrix","score":0.9227573871612549},{"id":"https://openalex.org/keywords/krylov-subspace","display_name":"Krylov subspace","score":0.8835082054138184},{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.7965176105499268},{"id":"https://openalex.org/keywords/solver","display_name":"Solver","score":0.705426037311554},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6639326214790344},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.5779292583465576},{"id":"https://openalex.org/keywords/broyden\u2013fletcher\u2013goldfarb\u2013shanno-algorithm","display_name":"Broyden\u2013Fletcher\u2013Goldfarb\u2013Shanno algorithm","score":0.5676616430282593},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.49290037155151367},{"id":"https://openalex.org/keywords/subspace-topology","display_name":"Subspace topology","score":0.4703747630119324},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.44935983419418335},{"id":"https://openalex.org/keywords/sampling","display_name":"Sampling (signal processing)","score":0.4309900403022766},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.396684467792511},{"id":"https://openalex.org/keywords/applied-mathematics","display_name":"Applied mathematics","score":0.3823229670524597},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.3796907961368561},{"id":"https://openalex.org/keywords/iterative-method","display_name":"Iterative method","score":0.29733067750930786},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.2485315203666687},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.24541813135147095}],"concepts":[{"id":"https://openalex.org/C203616005","wikidata":"https://www.wikidata.org/wiki/Q620495","display_name":"Hessian matrix","level":2,"score":0.9227573871612549},{"id":"https://openalex.org/C147060835","wikidata":"https://www.wikidata.org/wiki/Q1757151","display_name":"Krylov subspace","level":3,"score":0.8835082054138184},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.7965176105499268},{"id":"https://openalex.org/C2778770139","wikidata":"https://www.wikidata.org/wiki/Q1966904","display_name":"Solver","level":2,"score":0.705426037311554},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6639326214790344},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.5779292583465576},{"id":"https://openalex.org/C132721684","wikidata":"https://www.wikidata.org/wiki/Q2877013","display_name":"Broyden\u2013Fletcher\u2013Goldfarb\u2013Shanno algorithm","level":3,"score":0.5676616430282593},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.49290037155151367},{"id":"https://openalex.org/C32834561","wikidata":"https://www.wikidata.org/wiki/Q660730","display_name":"Subspace topology","level":2,"score":0.4703747630119324},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.44935983419418335},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.4309900403022766},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.396684467792511},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.3823229670524597},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.3796907961368561},{"id":"https://openalex.org/C159694833","wikidata":"https://www.wikidata.org/wiki/Q2321565","display_name":"Iterative method","level":2,"score":0.29733067750930786},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2485315203666687},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.24541813135147095},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.0},{"id":"https://openalex.org/C151319957","wikidata":"https://www.wikidata.org/wiki/Q752739","display_name":"Asynchronous communication","level":2,"score":0.0},{"id":"https://openalex.org/C106131492","wikidata":"https://www.wikidata.org/wiki/Q3072260","display_name":"Filter (signal processing)","level":2,"score":0.0},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0},{"id":"https://openalex.org/C50522688","wikidata":"https://www.wikidata.org/wiki/Q189833","display_name":"Economic growth","level":1,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.1109/asru.2013.6707747","is_oa":false,"landing_page_url":"https://doi.org/10.1109/asru.2013.6707747","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2013 IEEE Workshop on Automatic Speech Recognition and Understanding","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:1309.1508","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1309.1508","pdf_url":"https://arxiv.org/pdf/1309.1508","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:2049074570","is_oa":true,"landing_page_url":"https://arxiv.org/pdf/1309.1508.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.753.7982","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.753.7982","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://arxiv.org/pdf/1309.1508.pdf","raw_type":"text"},{"id":"doi:10.48550/arxiv.1309.1508","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1309.1508","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:1309.1508","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1309.1508","pdf_url":"https://arxiv.org/pdf/1309.1508","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":22,"referenced_works":["https://openalex.org/W196761320","https://openalex.org/W1575701986","https://openalex.org/W1751687266","https://openalex.org/W1967468998","https://openalex.org/W1972015017","https://openalex.org/W1987238397","https://openalex.org/W2006903949","https://openalex.org/W2051669046","https://openalex.org/W2061570747","https://openalex.org/W2083715026","https://openalex.org/W2098841537","https://openalex.org/W2114016253","https://openalex.org/W2116484487","https://openalex.org/W2130984546","https://openalex.org/W2155117693","https://openalex.org/W2403195671","https://openalex.org/W2762578791","https://openalex.org/W4236796448","https://openalex.org/W6608133726","https://openalex.org/W6643234761","https://openalex.org/W6674634876","https://openalex.org/W6684859321"],"related_works":["https://openalex.org/W2097428361","https://openalex.org/W2958624776","https://openalex.org/W2954388866","https://openalex.org/W2152009788","https://openalex.org/W3093154383","https://openalex.org/W2086368417","https://openalex.org/W3014076431","https://openalex.org/W2798028937","https://openalex.org/W2603900818","https://openalex.org/W2133819592","https://openalex.org/W2981670934","https://openalex.org/W2347734615","https://openalex.org/W2786704328","https://openalex.org/W2594963920","https://openalex.org/W3142207845","https://openalex.org/W2963015408","https://openalex.org/W3098728793","https://openalex.org/W2738307783","https://openalex.org/W2500884246","https://openalex.org/W2130336802"],"abstract_inverted_index":{"Hessian-free":[0,22],"training":[1],"has":[2],"become":[3],"a":[4,81,108,128,141,146,154],"popular":[5],"parallel":[6],"second":[7],"order":[8],"optimization":[9],"technique":[10],"for":[11,34,50,120],"Deep":[12],"Neural":[13],"Network":[14],"training.":[15],"This":[16],"study":[17],"aims":[18],"at":[19],"speeding":[20],"up":[21],"training,":[23,35],"both":[24],"by":[25],"means":[26],"of":[27,31,41,44,53,89,101,117],"decreasing":[28],"the":[29,42,54,68,72,87,96,115],"amount":[30,116],"data":[32,118],"used":[33,49],"as":[36,38,80,171],"well":[37],"through":[39],"reduction":[40],"number":[43],"Krylov":[45,91,123],"subspace":[46,92,124],"solver":[47],"iterations":[48],"implicit":[51],"estimation":[52],"Hessian.":[55],"In":[56],"this":[57],"paper,":[58],"we":[59,84,106,134],"develop":[60],"an":[61],"L-BFGS":[62,76],"based":[63],"preconditioning":[64],"scheme":[65],"that":[66,94,136,165],"avoids":[67],"need":[69],"to":[70],"access":[71],"Hessian":[73],"explicitly.":[74],"Since":[75],"cannot":[77],"be":[78],"regarded":[79],"fixed-point":[82],"iteration,":[83],"further":[85,167],"propose":[86,107],"employment":[88],"flexible":[90],"solvers":[93],"retain":[95],"desired":[97],"theoretical":[98],"convergence":[99],"guarantees":[100],"their":[102],"conventional":[103],"counterparts.":[104],"Second,":[105],"new":[109],"sampling":[110],"algorithm,":[111],"which":[112],"geometrically":[113],"increases":[114],"utilized":[119],"gradient":[121],"and":[122,174],"iteration":[125],"calculations.":[126],"On":[127],"50-hr":[129],"English":[130],"Broadcast":[131],"News":[132],"task,":[133,149],"find":[135],"these":[137,150],"methodologies":[138],"provide":[139,152],"roughly":[140],"1.5\u00d7":[142],"speed-up,":[143],"whereas,":[144],"on":[145],"300-hr":[147],"Switchboard":[148],"techniques":[151],"over":[153],"2.3\u00d7":[155],"speedup,":[156],"with":[157],"no":[158],"loss":[159],"in":[160],"WER.":[161],"These":[162],"results":[163],"suggest":[164],"even":[166],"speed-up":[168],"is":[169],"expected,":[170],"problems":[172],"scale":[173],"complexity":[175],"grows.":[176]},"counts_by_year":[{"year":2014,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}