{"id":"https://openalex.org/W4386546962","doi":"https://doi.org/10.1145/3622939","title":"Assessing Urdu Language Processing Tools via Statistical and Outlier Detection Methods on Urdu Tweets","display_name":"Assessing Urdu Language Processing Tools via Statistical and Outlier Detection Methods on Urdu Tweets","publication_year":2023,"publication_date":"2023-09-08","ids":{"openalex":"https://openalex.org/W4386546962","doi":"https://doi.org/10.1145/3622939"},"language":"en","primary_location":{"id":"doi:10.1145/3622939","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3622939","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3622939","source":{"id":"https://openalex.org/S4306421405","display_name":"ACM Transactions on Asian and Low-Resource Language Information Processing","issn_l":"2375-4699","issn":["2375-4699","2375-4702"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Asian and Low-Resource Language Information Processing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3622939","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5003090396","display_name":"Zoya","orcid":null},"institutions":[{"id":"https://openalex.org/I929597975","display_name":"National University of Sciences and Technology","ror":"https://ror.org/03w2j5y17","country_code":"PK","type":"education","lineage":["https://openalex.org/I929597975"]}],"countries":["PK"],"is_corresponding":false,"raw_author_name":"Zoya","raw_affiliation_strings":["School of Electrical Engineering and Computer Science (SEECS), National University of Sciences and Technology (NUST), Pakistan"],"raw_orcid":"https://orcid.org/0000-0002-9404-3903","affiliations":[{"raw_affiliation_string":"School of Electrical Engineering and Computer Science (SEECS), National University of Sciences and Technology (NUST), Pakistan","institution_ids":["https://openalex.org/I929597975"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024120099","display_name":"Seemab Latif","orcid":"https://orcid.org/0000-0002-5801-1568"},"institutions":[{"id":"https://openalex.org/I929597975","display_name":"National University of Sciences and Technology","ror":"https://ror.org/03w2j5y17","country_code":"PK","type":"education","lineage":["https://openalex.org/I929597975"]}],"countries":["PK"],"is_corresponding":false,"raw_author_name":"Seemab Latif","raw_affiliation_strings":["School of Electrical Engineering and Computer Science (SEECS), National University of Sciences and Technology (NUST), Pakistan"],"raw_orcid":"https://orcid.org/0000-0002-5801-1568","affiliations":[{"raw_affiliation_string":"School of Electrical Engineering and Computer Science (SEECS), National University of Sciences and Technology (NUST), Pakistan","institution_ids":["https://openalex.org/I929597975"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076524479","display_name":"Rabia Latif","orcid":"https://orcid.org/0000-0001-5304-5948"},"institutions":[{"id":"https://openalex.org/I142024983","display_name":"Prince Sultan University","ror":"https://ror.org/053mqrf26","country_code":"SA","type":"education","lineage":["https://openalex.org/I142024983"]}],"countries":["SA"],"is_corresponding":false,"raw_author_name":"Rabia Latif","raw_affiliation_strings":["Prince Sultan University, Saudi Arabia"],"raw_orcid":"https://orcid.org/0000-0001-5304-5948","affiliations":[{"raw_affiliation_string":"Prince Sultan University, Saudi Arabia","institution_ids":["https://openalex.org/I142024983"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068733133","display_name":"Hammad Majeed","orcid":"https://orcid.org/0000-0002-7770-5899"},"institutions":[{"id":"https://openalex.org/I201384688","display_name":"National University of Computer and Emerging Sciences","ror":"https://ror.org/003eyb898","country_code":"PK","type":"education","lineage":["https://openalex.org/I201384688"]}],"countries":["PK"],"is_corresponding":false,"raw_author_name":"Hammad Majeed","raw_affiliation_strings":["FAST National University of Computer and Emerging Sciences, Pakistan"],"raw_orcid":"https://orcid.org/0000-0002-7770-5899","affiliations":[{"raw_affiliation_string":"FAST National University of Computer and Emerging Sciences, Pakistan","institution_ids":["https://openalex.org/I201384688"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5052880936","display_name":"Nor Shahida Mohd Jamail","orcid":"https://orcid.org/0000-0003-0392-6506"},"institutions":[{"id":"https://openalex.org/I142024983","display_name":"Prince Sultan University","ror":"https://ror.org/053mqrf26","country_code":"SA","type":"education","lineage":["https://openalex.org/I142024983"]}],"countries":["SA"],"is_corresponding":false,"raw_author_name":"Nor Shahida Mohd Jamail","raw_affiliation_strings":["Prince Sultan University, Saudi Arabia"],"raw_orcid":"https://orcid.org/0000-0003-0392-6506","affiliations":[{"raw_affiliation_string":"Prince Sultan University, Saudi Arabia","institution_ids":["https://openalex.org/I142024983"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.5692,"has_fulltext":true,"cited_by_count":12,"citation_normalized_percentile":{"value":0.86121137,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"22","issue":"10","first_page":"1","last_page":"31"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.868273913860321},{"id":"https://openalex.org/keywords/lexical-analysis","display_name":"Lexical analysis","score":0.806349515914917},{"id":"https://openalex.org/keywords/urdu","display_name":"Urdu","score":0.6974972486495972},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.6601295471191406},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6283875703811646},{"id":"https://openalex.org/keywords/lemmatisation","display_name":"Lemmatisation","score":0.5470354557037354},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.539738118648529},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4549911916255951},{"id":"https://openalex.org/keywords/text-processing","display_name":"Text processing","score":0.4495495557785034},{"id":"https://openalex.org/keywords/named-entity-recognition","display_name":"Named-entity recognition","score":0.42972031235694885},{"id":"https://openalex.org/keywords/outlier","display_name":"Outlier","score":0.4196486473083496},{"id":"https://openalex.org/keywords/normalization","display_name":"Normalization (sociology)","score":0.4194514751434326},{"id":"https://openalex.org/keywords/lexical-database","display_name":"Lexical database","score":0.41305720806121826},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.10343185067176819},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.09754350781440735}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.868273913860321},{"id":"https://openalex.org/C176982825","wikidata":"https://www.wikidata.org/wiki/Q835922","display_name":"Lexical analysis","level":2,"score":0.806349515914917},{"id":"https://openalex.org/C2777350258","wikidata":"https://www.wikidata.org/wiki/Q1617","display_name":"Urdu","level":2,"score":0.6974972486495972},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6601295471191406},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6283875703811646},{"id":"https://openalex.org/C161831844","wikidata":"https://www.wikidata.org/wiki/Q2554325","display_name":"Lemmatisation","level":2,"score":0.5470354557037354},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.539738118648529},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4549911916255951},{"id":"https://openalex.org/C2779500292","wikidata":"https://www.wikidata.org/wiki/Q14802672","display_name":"Text processing","level":2,"score":0.4495495557785034},{"id":"https://openalex.org/C2779135771","wikidata":"https://www.wikidata.org/wiki/Q403574","display_name":"Named-entity recognition","level":3,"score":0.42972031235694885},{"id":"https://openalex.org/C79337645","wikidata":"https://www.wikidata.org/wiki/Q779824","display_name":"Outlier","level":2,"score":0.4196486473083496},{"id":"https://openalex.org/C136886441","wikidata":"https://www.wikidata.org/wiki/Q926129","display_name":"Normalization (sociology)","level":2,"score":0.4194514751434326},{"id":"https://openalex.org/C2780403423","wikidata":"https://www.wikidata.org/wiki/Q6537700","display_name":"Lexical database","level":3,"score":0.41305720806121826},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.10343185067176819},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.09754350781440735},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C157659113","wikidata":"https://www.wikidata.org/wiki/Q533822","display_name":"WordNet","level":2,"score":0.0},{"id":"https://openalex.org/C19165224","wikidata":"https://www.wikidata.org/wiki/Q23404","display_name":"Anthropology","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3622939","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3622939","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3622939","source":{"id":"https://openalex.org/S4306421405","display_name":"ACM Transactions on Asian and Low-Resource Language Information Processing","issn_l":"2375-4699","issn":["2375-4699","2375-4702"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Asian and Low-Resource Language Information Processing","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3622939","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3622939","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3622939","source":{"id":"https://openalex.org/S4306421405","display_name":"ACM Transactions on Asian and Low-Resource Language Information Processing","issn_l":"2375-4699","issn":["2375-4699","2375-4702"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Asian and Low-Resource Language Information Processing","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320327702","display_name":"Prince Sultan University","ror":"https://ror.org/053mqrf26"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4386546962.pdf","grobid_xml":"https://content.openalex.org/works/W4386546962.grobid-xml"},"referenced_works_count":35,"referenced_works":["https://openalex.org/W1623768750","https://openalex.org/W1972978214","https://openalex.org/W1973429805","https://openalex.org/W2016944307","https://openalex.org/W2038043464","https://openalex.org/W2085177100","https://openalex.org/W2164777277","https://openalex.org/W2409439155","https://openalex.org/W2618461247","https://openalex.org/W2899726877","https://openalex.org/W2942108662","https://openalex.org/W2976388333","https://openalex.org/W3011280467","https://openalex.org/W3021810042","https://openalex.org/W3025955883","https://openalex.org/W3037109418","https://openalex.org/W3044058576","https://openalex.org/W3089144988","https://openalex.org/W3145501851","https://openalex.org/W3168093350","https://openalex.org/W3172225542","https://openalex.org/W3176558250","https://openalex.org/W3186170310","https://openalex.org/W3194774163","https://openalex.org/W3199569609","https://openalex.org/W3201405160","https://openalex.org/W3201449892","https://openalex.org/W4229977739","https://openalex.org/W4239657535","https://openalex.org/W4240301789","https://openalex.org/W4247128285","https://openalex.org/W4319840113","https://openalex.org/W4380685549","https://openalex.org/W4404783704","https://openalex.org/W6683955732"],"related_works":["https://openalex.org/W2574640638","https://openalex.org/W2202496758","https://openalex.org/W3089211180","https://openalex.org/W3217218495","https://openalex.org/W4386058729","https://openalex.org/W4282591925","https://openalex.org/W3200555786","https://openalex.org/W3130974824","https://openalex.org/W4386546962","https://openalex.org/W2760600413"],"abstract_inverted_index":{"Text":[0],"pre-processing":[1,30,133],"is":[2],"a":[3,25,131],"crucial":[4],"step":[5],"in":[6,28,60,85,88,151,220,251],"Natural":[7],"Language":[8,90],"Processing":[9,91],"(NLP)":[10],"applications,":[11],"particularly":[12],"for":[13,46,55,108,126,147],"handling":[14,224],"informal":[15,226],"and":[16,38,71,77,101,117,143,168,178,190,196,204,211,229,231,243,269,278,304],"noisy":[17],"content":[18],"on":[19,186],"social":[20],"media.":[21],"Word-level":[22],"tokenization":[23,59,201],"plays":[24],"vital":[26],"role":[27],"text":[29],"by":[31],"removing":[32],"stop":[33],"words,":[34],"filtering":[35],"irrelevant":[36],"characters,":[37],"retaining":[39],"relevant":[40],"tokens.":[41,233],"These":[42],"tokens":[43,87,246],"are":[44],"essential":[45],"constructing":[47],"meaningful":[48],"n-grams":[49],"within":[50],"advanced":[51],"NLP":[52],"frameworks":[53],"used":[54],"data":[56,132],"modeling.":[57],"However,":[58,307],"low-resource":[61],"languages":[62],"like":[63],"Urdu":[64,89,162],"presents":[65],"challenges":[66],"due":[67],"to":[68,105,199],"language":[69,95,227],"complexity":[70],"limited":[72],"resources.":[73],"Conventional":[74],"space-based":[75,296],"methods":[76],"direct":[78],"application":[79],"of":[80,173,225,261],"language-specific":[81,100,175,188],"tools":[82],"often":[83],"result":[84],"erroneous":[86],"(ULP).":[92],"This":[93],"hinders":[94],"models":[96],"from":[97],"effectively":[98],"learning":[99],"domain-specific":[102],"tokens,":[103,228],"leading":[104],"sub-optimal":[106],"results":[107,202,275,299],"downstream":[109],"tasks":[110],"such":[111],"as":[112],"aspect":[113],"mining,":[114],"topic":[115,262,310,317],"modeling,":[116],"Named":[118],"Entity":[119],"Recognition":[120],"(NER).":[121],"To":[122],"address":[123],"this":[124],"issue":[125],"Urdu,":[127],"we":[128,191],"have":[129,157],"proposed":[130,144,208,240],"technique":[134],"that":[135],"detects":[136],"outliers":[137],"using":[138,164,264,281],"the":[139,165,170,187,207,217,239,248,259,294,298,308],"Inter-Quartile-Range":[140],"(IQR)":[141],"method":[142],"normalization":[145,212,244],"algorithms":[146],"creating":[148],"useful":[149],"lexicons":[150],"conjunction":[152],"with":[153,238,247,312,319],"existing":[154,174],"technologies.":[155],"We":[156],"collected":[158],"approximately":[159],"50":[160],"million":[161],"tweets":[163],"Twitter":[166],"API":[167],"conducted":[169],"performance":[171],"analysis":[172,194],"tokenizers":[176],"(Urduhack":[177],"Space-based":[179],"tokenizer).":[180],"Dataset":[181],"variants":[182],"were":[183],"created":[184],"based":[185],"tokenizers,":[189],"performed":[192],"statistical":[193],"tests":[195],"visualization":[197],"techniques":[198],"compare":[200],"before":[203],"after":[205],"applying":[206],"outlier":[209,241],"detection":[210,242],"method.":[213],"Our":[214],"findings":[215],"highlighted":[216],"noticeable":[218],"improvement":[219],"token":[221],"size":[222],"distributions,":[223],"misspelled":[230],"lengthy":[232],"The":[234,274],"Urduhack":[235],"tokenizer":[236],"combined":[237],"yielded":[245],"best-fitted":[249],"distribution":[250],"ULP.":[252],"Its":[253],"effectiveness":[254],"has":[255],"been":[256],"evaluated":[257],"through":[258],"task":[260],"modeling":[263,311,318],"Non-negative":[265],"Matrix":[266],"Factorization":[267],"(NMF)":[268],"Latent":[270],"Dirichlet":[271],"allocation":[272],"(LDA).":[273],"demonstrated":[276,301],"new":[277],"distinct":[279],"topics":[280,288],"unigram":[282],"features":[283,314],"while":[284],"achieving":[285],"highly":[286],"coherent":[287],"when":[289],"utilizing":[290],"bigram":[291,313,320],"features.":[292,321],"For":[293],"traditional":[295],"method,":[297],"consistently":[300],"improved":[302],"coherence":[303],"precision":[305],"scores.":[306],"NMF":[309],"outperformed":[315],"LDA":[316]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":5}],"updated_date":"2026-08-01T09:00:35.917206","created_date":"2025-10-10T00:00:00"}