{"id":"https://openalex.org/W7118497292","doi":"https://doi.org/10.48550/arxiv.2601.01091","title":"ks-lit-3m: A 3.1 million word kashmiri text dataset for large language model pretraining","display_name":"ks-lit-3m: A 3.1 million word kashmiri text dataset for large language model pretraining","publication_year":2026,"publication_date":"2026-01-03","ids":{"openalex":"https://openalex.org/W7118497292","doi":"https://doi.org/10.48550/arxiv.2601.01091"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2601.01091","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2601.01091","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2601.01091","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5115706324","display_name":"Haq Nawaz Malik","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Malik, Haq Nawaz","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5115706324"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.2531999945640564,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.2531999945640564,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.11599999666213989,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.10480000078678131,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/kashmiri","display_name":"Kashmiri","score":0.7318000197410583},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.6991000175476074},{"id":"https://openalex.org/keywords/fluency","display_name":"Fluency","score":0.5386000275611877},{"id":"https://openalex.org/keywords/word","display_name":"Word (group theory)","score":0.498199999332428},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.4440999925136566},{"id":"https://openalex.org/keywords/preprocessor","display_name":"Preprocessor","score":0.3865000009536743},{"id":"https://openalex.org/keywords/computational-linguistics","display_name":"Computational linguistics","score":0.365200012922287},{"id":"https://openalex.org/keywords/matching","display_name":"Matching (statistics)","score":0.3596000075340271}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7692999839782715},{"id":"https://openalex.org/C2776828431","wikidata":"https://www.wikidata.org/wiki/Q33552","display_name":"Kashmiri","level":3,"score":0.7318000197410583},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.7281000018119812},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.6991000175476074},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.65829998254776},{"id":"https://openalex.org/C2777413886","wikidata":"https://www.wikidata.org/wiki/Q3276013","display_name":"Fluency","level":2,"score":0.5386000275611877},{"id":"https://openalex.org/C90805587","wikidata":"https://www.wikidata.org/wiki/Q10944557","display_name":"Word (group theory)","level":2,"score":0.498199999332428},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.4440999925136566},{"id":"https://openalex.org/C34736171","wikidata":"https://www.wikidata.org/wiki/Q918333","display_name":"Preprocessor","level":2,"score":0.3865000009536743},{"id":"https://openalex.org/C155092808","wikidata":"https://www.wikidata.org/wiki/Q182557","display_name":"Computational linguistics","level":2,"score":0.365200012922287},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.3596000075340271},{"id":"https://openalex.org/C2474386","wikidata":"https://www.wikidata.org/wiki/Q461183","display_name":"Text corpus","level":2,"score":0.357699990272522},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.3310000002384186},{"id":"https://openalex.org/C129792486","wikidata":"https://www.wikidata.org/wiki/Q1050419","display_name":"Language identification","level":3,"score":0.32499998807907104},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.3052999973297119},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.2924000024795532},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.29120001196861267},{"id":"https://openalex.org/C171686336","wikidata":"https://www.wikidata.org/wiki/Q3532085","display_name":"Topic model","level":2,"score":0.2842000126838684},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.28290000557899475},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.2825999855995178},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.2808000147342682},{"id":"https://openalex.org/C117884012","wikidata":"https://www.wikidata.org/wiki/Q94489","display_name":"n-gram","level":3,"score":0.272599995136261},{"id":"https://openalex.org/C2780861071","wikidata":"https://www.wikidata.org/wiki/Q1062934","display_name":"Character (mathematics)","level":2,"score":0.26109999418258667},{"id":"https://openalex.org/C175293574","wikidata":"https://www.wikidata.org/wiki/Q697133","display_name":"Word lists by frequency","level":3,"score":0.2502000033855438}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2601.01091","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2601.01091","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2601.01091","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2601.01091","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[{"score":0.8502784967422485,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"demonstrate":[4],"remarkable":[5],"fluency":[6],"across":[7],"high-resource":[8],"languages":[9],"yet":[10],"consistently":[11],"fail":[12],"to":[13,51,56,108,176],"generate":[14],"coherent":[15],"text":[16,97],"in":[17,59,179],"Kashmiri,":[18],"a":[19,38,70,93,123,160],"language":[20,84,102,166,182],"spoken":[21],"by":[22,128],"approximately":[23],"seven":[24],"million":[25,75,78],"people.":[26],"This":[27,66],"performance":[28],"disparity":[29],"stems":[30],"not":[31],"from":[32,37,112,145],"inherent":[33],"model":[34,103],"limitations":[35],"but":[36],"critical":[39],"scarcity":[40],"of":[41,46,73,122],"high-quality":[42],"training":[43,104],"data.":[44],"Decades":[45],"Kashmiri":[47,165,180],"literature":[48],"remain":[49],"inaccessible":[50],"modern":[52],"NLP":[53],"pipelines":[54],"due":[55],"their":[57],"encoding":[58],"the":[60,120,173],"proprietary":[61],"InPage":[62],"desktop":[63],"publishing":[64],"format.":[65],"paper":[67],"introduces":[68],"KS-LIT-3M,":[69],"curated":[71],"corpus":[72,116],"3.1":[74],"words":[76,143],"(16.4":[77],"characters)":[79],"specifically":[80],"designed":[81],"for":[82,100,164],"pretraining":[83],"models":[85,106],"on":[86],"Kashmiri.":[87],"The":[88,115,168],"dataset":[89,169],"is":[90,170],"structured":[91],"as":[92],"single":[94],"continuous":[95],"linear":[96],"stream,":[98],"optimized":[99],"causal":[101],"where":[105],"learn":[107],"predict":[109],"subsequent":[110],"tokens":[111],"preceding":[113],"context.":[114],"was":[117],"constructed":[118],"through":[119],"development":[121],"specialized":[124],"InPage-to-Unicode":[125],"converter,":[126],"followed":[127],"rigorous":[129],"preprocessing":[130],"including":[131,148],"English":[132],"contamination":[133],"removal,":[134],"character":[135],"normalization,":[136],"and":[137,155],"quality":[138],"validation.":[139],"Encompassing":[140],"131,607":[141],"unique":[142],"drawn":[144],"diverse":[146],"genres":[147],"literary":[149],"works,":[150],"journalistic":[151],"writing,":[152],"academic":[153],"texts,":[154],"religious":[156],"scholarship,":[157],"KS-LIT-3M":[158],"addresses":[159],"fundamental":[161],"resource":[162],"gap":[163],"technology.":[167],"released":[171],"under":[172],"CC-BY-4.0":[174],"license":[175],"facilitate":[177],"research":[178],"natural":[181],"processing.":[183]},"counts_by_year":[],"updated_date":"2026-01-08T20:10:11.968330","created_date":"2026-01-08T00:00:00"}