{"id":"https://openalex.org/W4385572040","doi":"https://doi.org/10.18653/v1/2023.sustainlp-1.11","title":"Can we Pretrain a SotA Legal Language Model on a Budget From Scratch?","display_name":"Can we Pretrain a SotA Legal Language Model on a Budget From Scratch?","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4385572040","doi":"https://doi.org/10.18653/v1/2023.sustainlp-1.11"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2023.sustainlp-1.11","doi":"https://doi.org/10.18653/v1/2023.sustainlp-1.11","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.sustainlp-1.11","pdf_url":"https://aclanthology.org/2023.sustainlp-1.11.pdf","source":{"id":"https://openalex.org/S7407107969","display_name":"Workshop on Simple and Efficient Natural Language Processing (SustaiNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of The Fourth Workshop on Simple and Efficient Natural Language Processing (SustaiNLP)","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2023.sustainlp-1.11.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5063537470","display_name":"Joel Niklaus","orcid":"https://orcid.org/0000-0002-2779-1653","observed_orcids":["https://orcid.org/0000-0002-2779-1653"]},"institutions":[{"id":"https://openalex.org/I4210121390","display_name":"Thomson Reuters (Switzerland)","ror":"https://ror.org/02kh7ez55","country_code":"CH","type":"company","lineage":["https://openalex.org/I4210121390","https://openalex.org/I68384125"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Joel Niklaus","raw_affiliation_strings":["Thomson Reuters Labs, Zug, Switzerland","Social Science Research Network, Rochester, NY"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Thomson Reuters Labs, Zug, Switzerland","institution_ids":["https://openalex.org/I4210121390"]},{"raw_affiliation_string":"Social Science Research Network, Rochester, NY","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5088812866","display_name":"Daniele Giofr\u00e9","orcid":"https://orcid.org/0000-0002-8211-9392","observed_orcids":["https://orcid.org/0000-0002-8211-9392"]},"institutions":[{"id":"https://openalex.org/I4210121390","display_name":"Thomson Reuters (Switzerland)","ror":"https://ror.org/02kh7ez55","country_code":"CH","type":"company","lineage":["https://openalex.org/I4210121390","https://openalex.org/I68384125"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Daniele Giofre","raw_affiliation_strings":["Thomson Reuters Labs, Zug, Switzerland","Social Science Research Network, Rochester, NY"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Thomson Reuters Labs, Zug, Switzerland","institution_ids":["https://openalex.org/I4210121390"]},{"raw_affiliation_string":"Social Science Research Network, Rochester, NY","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I4210121390"],"apc_list":null,"apc_paid":null,"fwci":0.4534,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":{"value":0.64014046,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":93,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"158","last_page":"182"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9682999849319458,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9682999849319458,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.0215000007301569,"subfield":{"id":"https://openalex.org/subfields/3308","display_name":"Law"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.005799999926239252,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/longformer","display_name":"Longformer","score":0.9290000200271606},{"id":"https://openalex.org/keywords/legal-document-summarization","display_name":"legal text summarization","score":0.6420000195503235},{"id":"https://openalex.org/keywords/pubmed","display_name":"PubMed","score":0.32100000977516174},{"id":"https://openalex.org/keywords/domain-specific-language-models","display_name":"domain-specific language models","score":0.5680000185966492}],"concepts":[{"id":"https://openalex.org/C2781235140","wikidata":"https://www.wikidata.org/wiki/Q275131","display_name":"Scratch","level":2,"score":0.7747818231582642},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7188261151313782},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3683125376701355},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.36273306608200073},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.32969725131988525},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.2829664349555969}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.18653/v1/2023.sustainlp-1.11","doi":"https://doi.org/10.18653/v1/2023.sustainlp-1.11","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.sustainlp-1.11","pdf_url":"https://aclanthology.org/2023.sustainlp-1.11.pdf","source":{"id":"https://openalex.org/S7407107969","display_name":"Workshop on Simple and Efficient Natural Language Processing (SustaiNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of The Fourth Workshop on Simple and Efficient Natural Language Processing (SustaiNLP)","raw_type":"proceedings-article"},{"id":"pmh:oai:arbor.bfh.ch:19709","doi":null,"is_oa":true,"landing_page_url":"https://arbor.bfh.ch/19709/","pdf_url":null,"source":{"id":"https://openalex.org/S4406922806","display_name":"ARBOR - Bern University of Applied Sciences Repository","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Niklaus, Jo\u00ebl; Giofr\u00e9, Daniele (13 July 2023). Can we Pretrain a SotA Legal Language Model on a Budget From Scratch? In: Proceedings of The Fourth Workshop on Simple and Efficient Natural Language Processing (SustaiNLP) (pp. 158-182). Association for Computational Linguistics 10.18653/v1/2023.sustainlp-1.11 <http://dx.doi.org/10.18653/v1/2023.sustainlp-1.11>","raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":{"id":"doi:10.18653/v1/2023.sustainlp-1.11","doi":"https://doi.org/10.18653/v1/2023.sustainlp-1.11","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.sustainlp-1.11","pdf_url":"https://aclanthology.org/2023.sustainlp-1.11.pdf","source":{"id":"https://openalex.org/S7407107969","display_name":"Workshop on Simple and Efficient Natural Language Processing (SustaiNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of The Fourth Workshop on Simple and Efficient Natural Language Processing (SustaiNLP)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4385572040.pdf","grobid_xml":"https://content.openalex.org/works/W4385572040.grobid-xml"},"referenced_works_count":38,"referenced_works":["https://openalex.org/W2626778328","https://openalex.org/W2911489562","https://openalex.org/W2923014074","https://openalex.org/W2936695845","https://openalex.org/W2948223045","https://openalex.org/W2948586190","https://openalex.org/W2950813464","https://openalex.org/W2953280096","https://openalex.org/W2965373594","https://openalex.org/W2970771982","https://openalex.org/W2978041899","https://openalex.org/W2979826702","https://openalex.org/W2982399380","https://openalex.org/W2996264288","https://openalex.org/W2997517014","https://openalex.org/W2998353611","https://openalex.org/W3015468748","https://openalex.org/W3016339201","https://openalex.org/W3021293129","https://openalex.org/W3045733172","https://openalex.org/W3085139254","https://openalex.org/W3092129462","https://openalex.org/W3103682594","https://openalex.org/W3156891177","https://openalex.org/W3162090017","https://openalex.org/W3177382889","https://openalex.org/W3203176827","https://openalex.org/W3212554815","https://openalex.org/W4221143384","https://openalex.org/W4287183852","https://openalex.org/W4318719533","https://openalex.org/W4320005767","https://openalex.org/W4385567137","https://openalex.org/W4385570763","https://openalex.org/W4385573839","https://openalex.org/W4385574011","https://openalex.org/W4404815084","https://openalex.org/W7133187895"],"related_works":["https://openalex.org/W4310561710","https://openalex.org/W4385574011","https://openalex.org/W4388585297","https://openalex.org/W3198156039","https://openalex.org/W3177382889","https://openalex.org/W4386566549","https://openalex.org/W7126034337","https://openalex.org/W7220134018","https://openalex.org/W3199522085","https://openalex.org/W3212522196"],"abstract_inverted_index":{"Even":[0],"though":[1],"many":[2],"efficient":[3,83,94],"transformers":[4],"have":[5],"been":[6],"proposed,":[7],"only":[8,38],"few":[9],"such":[10],"models":[11,80,107,127,146],"are":[12],"available":[13],"for":[14,150],"specialized":[15],"domains.Additionally,":[16],"since":[17,66],"the":[18,31,51,67,82,105,113,123,132],"pretraining":[19,48,93],"process":[20],"is":[21,36,50,96],"extremely":[22],"costly":[23],"in":[24,39,139],"general":[25],"-but":[26],"even":[27],"more":[28,59],"so":[29],"as":[30,147],"sequence":[32],"length":[33],"increases":[34],"-it":[35],"often":[37],"reach":[40],"of":[41,46],"large":[42],"research":[43],"labs.One":[44],"way":[45],"making":[47],"cheaper":[49],"Replaced":[52],"Token":[53],"Detection":[54],"(RTD)":[55],"task,":[56],"by":[57],"providing":[58],"signal":[60],"during":[61],"training":[62],"compared":[63],"to":[64,90,115],"MLM,":[65],"loss":[68],"can":[69],"be":[70],"computed":[71],"over":[72],"all":[73],"tokens.In":[74],"this":[75],"work,":[76],"we":[77],"train":[78],"Longformer":[79],"with":[81],"RTD":[84],"task":[85],"on":[86,108,131],"long-context":[87],"legal":[88],"data":[89],"showcase":[91],"that":[92,121],"LMs":[95],"possible":[97],"using":[98],"less":[99],"than":[100],"12":[101],"GPU":[102],"days.We":[103],"evaluate":[104],"trained":[106],"challenging":[109],"summarization":[110],"tasks":[111,138],"requiring":[112],"model":[114],"summarize":[116],"complex":[117],"long":[118],"texts.We":[119],"find":[120],"both":[122],"small":[124],"and":[125,135,152],"base":[126],"outperform":[128],"their":[129,140],"baselines":[130],"in-domain":[133],"BillSum":[134],"out-of-domain":[136],"PubMed":[137],"respective":[141],"parameter":[142],"range.We":[143],"publish":[144],"our":[145],"a":[148],"resource":[149],"researchers":[151],"practitioners.":[153]},"counts_by_year":[{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":2}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2023-08-05T00:00:00"}