{"id":"https://openalex.org/W2741884846","doi":"https://doi.org/10.24963/ijcai.2017/586","title":"Fast Parallel Training of Neural Language Models","display_name":"Fast Parallel Training of Neural Language Models","publication_year":2017,"publication_date":"2017-07-28","ids":{"openalex":"https://openalex.org/W2741884846","doi":"https://doi.org/10.24963/ijcai.2017/586","mag":"2741884846"},"language":"en","primary_location":{"id":"doi:10.24963/ijcai.2017/586","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2017/586","pdf_url":"https://www.ijcai.org/proceedings/2017/0586.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twenty-Sixth International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.ijcai.org/proceedings/2017/0586.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100600701","display_name":"Tong Xiao","orcid":"https://orcid.org/0000-0002-5842-6501"},"institutions":[{"id":"https://openalex.org/I9224756","display_name":"Northeastern University","ror":"https://ror.org/03awzbc87","country_code":"CN","type":"education","lineage":["https://openalex.org/I9224756"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tong Xiao","raw_affiliation_strings":["Northeastern University, Shenyang 110819, China","NiuTrans Lab., Northeastern University, Shenyang 110819, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Northeastern University, Shenyang 110819, China","institution_ids":["https://openalex.org/I9224756"]},{"raw_affiliation_string":"NiuTrans Lab., Northeastern University, Shenyang 110819, China","institution_ids":["https://openalex.org/I9224756"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100370145","display_name":"Jingbo Zhu","orcid":null},"institutions":[{"id":"https://openalex.org/I9224756","display_name":"Northeastern University","ror":"https://ror.org/03awzbc87","country_code":"CN","type":"education","lineage":["https://openalex.org/I9224756"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jingbo Zhu","raw_affiliation_strings":["Northeastern University, Shenyang 110819, China","NiuTrans Lab., Northeastern University, Shenyang 110819, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Northeastern University, Shenyang 110819, China","institution_ids":["https://openalex.org/I9224756"]},{"raw_affiliation_string":"NiuTrans Lab., Northeastern University, Shenyang 110819, China","institution_ids":["https://openalex.org/I9224756"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012126405","display_name":"Tongran Liu","orcid":null},"institutions":[{"id":"https://openalex.org/I4210131870","display_name":"Institute of Psychology, Chinese Academy of Sciences","ror":"https://ror.org/03j7v5j15","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210131870"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tongran Liu","raw_affiliation_strings":["Institute of Psychology (CAS), Beijing 100101, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Psychology (CAS), Beijing 100101, China","institution_ids":["https://openalex.org/I4210131870"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101997180","display_name":"Chunliang Zhang","orcid":"https://orcid.org/0000-0002-4382-2413"},"institutions":[{"id":"https://openalex.org/I9224756","display_name":"Northeastern University","ror":"https://ror.org/03awzbc87","country_code":"CN","type":"education","lineage":["https://openalex.org/I9224756"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chunliang Zhang","raw_affiliation_strings":["Northeastern University, Shenyang 110819, China","NiuTrans Lab., Northeastern University, Shenyang 110819, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Northeastern University, Shenyang 110819, China","institution_ids":["https://openalex.org/I9224756"]},{"raw_affiliation_string":"NiuTrans Lab., Northeastern University, Shenyang 110819, China","institution_ids":["https://openalex.org/I9224756"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":9,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"4193","last_page":"4199"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9973000288009644,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.9512051343917847},{"id":"https://openalex.org/keywords/perplexity","display_name":"Perplexity","score":0.8945150971412659},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8679113388061523},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.7546789646148682},{"id":"https://openalex.org/keywords/asynchronous-communication","display_name":"Asynchronous communication","score":0.6121921539306641},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.557066798210144},{"id":"https://openalex.org/keywords/stochastic-gradient-descent","display_name":"Stochastic gradient descent","score":0.5360956788063049},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5133997201919556},{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.46373406052589417},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.456748366355896},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.3518657088279724},{"id":"https://openalex.org/keywords/computer-engineering","display_name":"Computer engineering","score":0.33443596959114075},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3330637216567993}],"concepts":[{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.9512051343917847},{"id":"https://openalex.org/C100279451","wikidata":"https://www.wikidata.org/wiki/Q372193","display_name":"Perplexity","level":3,"score":0.8945150971412659},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8679113388061523},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.7546789646148682},{"id":"https://openalex.org/C151319957","wikidata":"https://www.wikidata.org/wiki/Q752739","display_name":"Asynchronous communication","level":2,"score":0.6121921539306641},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.557066798210144},{"id":"https://openalex.org/C206688291","wikidata":"https://www.wikidata.org/wiki/Q7617819","display_name":"Stochastic gradient descent","level":3,"score":0.5360956788063049},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5133997201919556},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.46373406052589417},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.456748366355896},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3518657088279724},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.33443596959114075},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3330637216567993},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.0},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.24963/ijcai.2017/586","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2017/586","pdf_url":"https://www.ijcai.org/proceedings/2017/0586.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twenty-Sixth International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.24963/ijcai.2017/586","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2017/586","pdf_url":"https://www.ijcai.org/proceedings/2017/0586.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twenty-Sixth International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G444532422","display_name":null,"funder_award_id":"61672138","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G545337727","display_name":null,"funder_award_id":"61432013","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320335787","display_name":"Fundamental Research Funds for the Central Universities","ror":null}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2741884846.pdf","grobid_xml":"https://content.openalex.org/works/W2741884846.grobid-xml"},"referenced_works_count":25,"referenced_works":["https://openalex.org/W36903255","https://openalex.org/W179875071","https://openalex.org/W1558797106","https://openalex.org/W2049469158","https://openalex.org/W2064675550","https://openalex.org/W2095705004","https://openalex.org/W2113104171","https://openalex.org/W2120861206","https://openalex.org/W2127163275","https://openalex.org/W2131462252","https://openalex.org/W2138243089","https://openalex.org/W2138302120","https://openalex.org/W2144839971","https://openalex.org/W2152808281","https://openalex.org/W2166706236","https://openalex.org/W2259472270","https://openalex.org/W2407022425","https://openalex.org/W2463033603","https://openalex.org/W2949198759","https://openalex.org/W2951781666","https://openalex.org/W2952033860","https://openalex.org/W2963804082","https://openalex.org/W2998704965","https://openalex.org/W3015789953","https://openalex.org/W4285719527"],"related_works":["https://openalex.org/W2169518243","https://openalex.org/W2252095989","https://openalex.org/W4322096525","https://openalex.org/W2551914602","https://openalex.org/W4281893144","https://openalex.org/W4287323699","https://openalex.org/W2105076537","https://openalex.org/W2084531783","https://openalex.org/W2902731467","https://openalex.org/W2787311093"],"abstract_inverted_index":{"Training":[0],"neural":[1,86],"language":[2,88],"models":[3],"(NLMs)":[4],"is":[5,117],"very":[6],"time":[7,32],"consuming":[8],"and":[9],"we":[10,48],"need":[11],"parallelization":[12],"for":[13,38,57],"system":[14],"speedup.":[15],"However,":[16],"standard":[17,104],"training":[18,71],"methods":[19],"have":[20],"poor":[21],"scalability":[22],"across":[23],"multiple":[24],"devices":[25],"(e.g.,":[26],"GPUs)":[27],"due":[28],"to":[29,35,53],"the":[30,42,65,70,103,123],"huge":[31],"cost":[33],"required":[34],"transmit":[36],"data":[37,55],"gradient":[39,107],"sharing":[40],"in":[41,114],"back-propagation":[43],"process.":[44],"In":[45],"this":[46],"paper":[47],"present":[49],"a":[50,63,74,84,97],"sampling-based":[51],"approach":[52,78],"reducing":[54],"transmission":[56],"better":[58],"scaling":[59],"of":[60,99],"NLMs.":[61],"As":[62],"''bonus'',":[64],"resulting":[66],"model":[67],"also":[68],"improves":[69],"speed":[72,81],"on":[73,83],"single":[75,125],"device.":[76],"Our":[77],"yields":[79],"significant":[80],"improvements":[82],"recurrent":[85],"network-based":[87],"model.":[89],"On":[90],"four":[91],"NVIDIA":[92],"GTX1080":[93],"GPUs,":[94],"it":[95],"achieves":[96],"speedup":[98],"2.1+":[100],"times":[101,120],"over":[102],"asynchronous":[105],"stochastic":[106],"descent":[108],"baseline,":[109],"yet":[110],"with":[111],"no":[112],"increase":[113],"perplexity.":[115],"This":[116],"even":[118],"4.2":[119],"faster":[121],"than":[122],"naive":[124],"GPU":[126],"counterpart.":[127]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":2},{"year":2018,"cited_by_count":2},{"year":2017,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}