{"id":"https://openalex.org/W4200219015","doi":"https://doi.org/10.1109/candar53791.2021.00022","title":"Efficient and Large Scale Pre-training Techniques for Japanese Natural Language Processing","display_name":"Efficient and Large Scale Pre-training Techniques for Japanese Natural Language Processing","publication_year":2021,"publication_date":"2021-11-01","ids":{"openalex":"https://openalex.org/W4200219015","doi":"https://doi.org/10.1109/candar53791.2021.00022"},"language":"en","primary_location":{"id":"doi:10.1109/candar53791.2021.00022","doi":"https://doi.org/10.1109/candar53791.2021.00022","is_oa":false,"landing_page_url":"https://doi.org/10.1109/candar53791.2021.00022","pdf_url":null,"source":{"id":"https://openalex.org/S4306420119","display_name":"International Symposium on Computing and Networking","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 Ninth International Symposium on Computing and Networking (CANDAR)","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5089627100","display_name":"Akihiko Kasagi","orcid":"https://orcid.org/0000-0002-5793-335X","observed_orcids":["https://orcid.org/0000-0002-5793-335X"]},"institutions":[{"id":"https://openalex.org/I2252096349","display_name":"Fujitsu (Japan)","ror":"https://ror.org/038e2g226","country_code":"JP","type":"company","lineage":["https://openalex.org/I2252096349"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Akihiko Kasagi","raw_affiliation_strings":["Fujitsu Limited, Kawasaki, Kanagawa, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Fujitsu Limited, Kawasaki, Kanagawa, Japan","institution_ids":["https://openalex.org/I2252096349"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050432884","display_name":"Masahiro Asaoka","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I2252096349","display_name":"Fujitsu (Japan)","ror":"https://ror.org/038e2g226","country_code":"JP","type":"company","lineage":["https://openalex.org/I2252096349"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Masahiro Asaoka","raw_affiliation_strings":["Fujitsu Limited, Kawasaki, Kanagawa, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Fujitsu Limited, Kawasaki, Kanagawa, Japan","institution_ids":["https://openalex.org/I2252096349"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041051226","display_name":"Akihiro Tabuchi","orcid":"https://orcid.org/0009-0009-3931-3817","observed_orcids":["https://orcid.org/0009-0009-3931-3817"]},"institutions":[{"id":"https://openalex.org/I2252096349","display_name":"Fujitsu (Japan)","ror":"https://ror.org/038e2g226","country_code":"JP","type":"company","lineage":["https://openalex.org/I2252096349"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Akihiro Tabuchi","raw_affiliation_strings":["Fujitsu Limited, Kawasaki, Kanagawa, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Fujitsu Limited, Kawasaki, Kanagawa, Japan","institution_ids":["https://openalex.org/I2252096349"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001297674","display_name":"Yosuke Oyama","orcid":"https://orcid.org/0000-0001-8316-3291","observed_orcids":["https://orcid.org/0000-0001-8316-3291"]},"institutions":[{"id":"https://openalex.org/I2252096349","display_name":"Fujitsu (Japan)","ror":"https://ror.org/038e2g226","country_code":"JP","type":"company","lineage":["https://openalex.org/I2252096349"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Yosuke Oyama","raw_affiliation_strings":["Fujitsu Limited, Kawasaki, Kanagawa, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Fujitsu Limited, Kawasaki, Kanagawa, Japan","institution_ids":["https://openalex.org/I2252096349"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081391958","display_name":"Takumi Honda","orcid":"https://orcid.org/0000-0002-1572-4533","observed_orcids":["https://orcid.org/0000-0002-1572-4533","https://orcid.org/0009-0007-6056-0738"]},"institutions":[{"id":"https://openalex.org/I2252096349","display_name":"Fujitsu (Japan)","ror":"https://ror.org/038e2g226","country_code":"JP","type":"company","lineage":["https://openalex.org/I2252096349"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Takumi Honda","raw_affiliation_strings":["Fujitsu Limited, Kawasaki, Kanagawa, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Fujitsu Limited, Kawasaki, Kanagawa, Japan","institution_ids":["https://openalex.org/I2252096349"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039299164","display_name":"Yasufumi Sakai","orcid":"https://orcid.org/0000-0001-7135-9449","observed_orcids":["https://orcid.org/0000-0001-7135-9449"]},"institutions":[{"id":"https://openalex.org/I2252096349","display_name":"Fujitsu (Japan)","ror":"https://ror.org/038e2g226","country_code":"JP","type":"company","lineage":["https://openalex.org/I2252096349"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Yasufumi Sakai","raw_affiliation_strings":["Fujitsu Limited, Kawasaki, Kanagawa, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Fujitsu Limited, Kawasaki, Kanagawa, Japan","institution_ids":["https://openalex.org/I2252096349"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025755721","display_name":"Dang Duy Thang","orcid":"https://orcid.org/0000-0003-1348-7252","observed_orcids":["https://orcid.org/0000-0003-1348-7252"]},"institutions":[{"id":"https://openalex.org/I2252096349","display_name":"Fujitsu (Japan)","ror":"https://ror.org/038e2g226","country_code":"JP","type":"company","lineage":["https://openalex.org/I2252096349"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Thang Dang","raw_affiliation_strings":["Fujitsu Limited, Kawasaki, Kanagawa, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Fujitsu Limited, Kawasaki, Kanagawa, Japan","institution_ids":["https://openalex.org/I2252096349"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5034919372","display_name":"Tsuguchika Tabaru","orcid":"https://orcid.org/0000-0001-6568-1968","observed_orcids":["https://orcid.org/0000-0001-6568-1968"]},"institutions":[{"id":"https://openalex.org/I2252096349","display_name":"Fujitsu (Japan)","ror":"https://ror.org/038e2g226","country_code":"JP","type":"company","lineage":["https://openalex.org/I2252096349"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Tsuguchika Tabaru","raw_affiliation_strings":["Fujitsu Limited, Kawasaki, Kanagawa, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Fujitsu Limited, Kawasaki, Kanagawa, Japan","institution_ids":["https://openalex.org/I2252096349"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I2252096349"],"apc_list":null,"apc_paid":null,"fwci":0.1153,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.36260022,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":94},"biblio":{"volume":"33","issue":null,"first_page":"108","last_page":"113"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9854000210762024,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9854000210762024,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.012199999764561653,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.00039999998989515007,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pretraining","display_name":"pre-training","score":0.7070000171661377},{"id":"https://openalex.org/keywords/japanese-language","display_name":"Japanese language","score":0.5879999995231628},{"id":"https://openalex.org/keywords/bert","display_name":"BERT","score":0.6740000247955322},{"id":"https://openalex.org/keywords/learning-efficiency","display_name":"learning efficiency","score":0.6840000152587891},{"id":"https://openalex.org/keywords/fine-tuning","display_name":"fine-tuning","score":0.7089999914169312}],"concepts":[{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.8811373114585876},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8642793893814087},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.6156550645828247},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5389829874038696},{"id":"https://openalex.org/C79974875","wikidata":"https://www.wikidata.org/wiki/Q483639","display_name":"Cloud computing","level":2,"score":0.5114132761955261},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.49428772926330566},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4883520305156708},{"id":"https://openalex.org/C174348530","wikidata":"https://www.wikidata.org/wiki/Q188635","display_name":"Bridging (networking)","level":2,"score":0.47390449047088623},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4662809371948242},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.45391517877578735},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4298710823059082},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.1759631335735321},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.08280214667320251},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/candar53791.2021.00022","doi":"https://doi.org/10.1109/candar53791.2021.00022","is_oa":false,"landing_page_url":"https://doi.org/10.1109/candar53791.2021.00022","pdf_url":null,"source":{"id":"https://openalex.org/S4306420119","display_name":"International Symposium on Computing and Networking","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 Ninth International Symposium on Computing and Networking (CANDAR)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320311508","display_name":"National Institute of Advanced Industrial Science and Technology","ror":"https://ror.org/01703db54"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":21,"referenced_works":["https://openalex.org/W1840435438","https://openalex.org/W2183341477","https://openalex.org/W2185175083","https://openalex.org/W2622263826","https://openalex.org/W2884711234","https://openalex.org/W2885185669","https://openalex.org/W2896457183","https://openalex.org/W2901541570","https://openalex.org/W2920668770","https://openalex.org/W2923014074","https://openalex.org/W2946328221","https://openalex.org/W2952370363","https://openalex.org/W2962784628","https://openalex.org/W2965373594","https://openalex.org/W2973727699","https://openalex.org/W2975429091","https://openalex.org/W2990957769","https://openalex.org/W2996035354","https://openalex.org/W3030163527","https://openalex.org/W3037115325","https://openalex.org/W6771917389"],"related_works":["https://openalex.org/W3046354018","https://openalex.org/W4308900169","https://openalex.org/W3120519792","https://openalex.org/W3155183406","https://openalex.org/W3037115325","https://openalex.org/W4394962103","https://openalex.org/W4310561961","https://openalex.org/W4309442582","https://openalex.org/W7203681626","https://openalex.org/W2978017171"],"abstract_inverted_index":{"Pre-training":[0],"in":[1,17,42,53,107,163],"natural":[2],"language":[3,139],"processing":[4],"greatly":[5],"affects":[6],"the":[7,18,31,64,83,91,121,128,153,189],"accuracy":[8,190,207,215],"of":[9,75,85,105,114,130,191,208,216],"downstream":[10,192],"tasks.":[11],"However,":[12],"pre-training":[13,51,162,167],"is":[14,45,68,79],"a":[15,26,48,54,58,98,111,137,174,201],"bottleneck":[16],"AI":[19,145],"system":[20],"development":[21],"process":[22],"because":[23],"it":[24,78],"takes":[25,168],"long":[27],"time":[28,56,65,132],"to":[29,46,81,109,187],"train":[30],"neural":[32],"network":[33],"model":[34,52],"by":[35],"using":[36,57,72,120,141],"large-scale":[37,59,115],"input":[38],"data.":[39],"Our":[40,150],"purpose":[41],"this":[43],"paper":[44],"obtain":[47],"highly":[49],"accurate":[50],"short":[55],"computation":[60,76],"environment.":[61],"Since":[62],"reducing":[63],"per":[66,94,156],"iteration":[67,95,157],"difficult":[69],"even":[70],"if":[71,171],"large":[73],"number":[74,84],"nodes,":[77],"necessary":[80],"reduce":[82],"iterations.":[86],"Therefore,":[87],"we":[88,172],"focus":[89],"on":[90,124,144],"learning":[92,131,134,154],"efficiency":[93,135,155],"and":[96,126,133,160,183,197,210],"choice":[97],"dense":[99,122],"Masked":[100],"Language":[101],"Model":[102],"(MLM)":[103],"task":[104],"pretraining":[106],"order":[108],"utilize":[110],"significant":[112],"power":[113],"cluster.":[116],"We":[117,177],"implemented":[118],"BERT-xlarge":[119,151,212],"MLM":[123],"Megatron-LM":[125],"evaluated":[127,179],"improvement":[129],"for":[136],"Japanese":[138],"dataset":[140],"768":[142],"GPUs":[143],"Bridging":[146],"Cloud":[147],"Infrastructure":[148],"(ABCI).":[149],"improves":[152],"10":[158],"times":[159],"completes":[161],"4.65":[164],"hours.":[165],"This":[166],"4.9":[169],"months":[170],"use":[173],"single":[175],"GPU.":[176],"also":[178],"two":[180],"fine-tunings,":[181],"JSNLI":[182],"Twitter":[184,217],"evaluation":[185,218],"analysis,":[186],"compare":[188],"tasks":[193],"with":[194],"our":[195,203,211],"BERTs":[196],"other":[198],"BERTs.":[199],"As":[200],"result,":[202],"BERT-3.9b":[204],"achieved":[205,213],"94.30%":[206],"JSNLI,":[209],"90.63%":[214],"analysis.":[219]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2022,"cited_by_count":1}],"updated_date":"2026-10-29T09:56:32.521200","created_date":"2021-12-31T00:00:00"}