{"id":"https://openalex.org/W4403730655","doi":"https://doi.org/10.52700/scir.v6i1.157","title":"A Systematic Review of Attention Models in Natural Language Processing","display_name":"A Systematic Review of Attention Models in Natural Language Processing","publication_year":2024,"publication_date":"2024-06-30","ids":{"openalex":"https://openalex.org/W4403730655","doi":"https://doi.org/10.52700/scir.v6i1.157"},"language":"en","primary_location":{"id":"doi:10.52700/scir.v6i1.157","doi":"https://doi.org/10.52700/scir.v6i1.157","is_oa":true,"landing_page_url":"https://doi.org/10.52700/scir.v6i1.157","pdf_url":"https://scir.wum.edu.pk/index.php/ojs/article/download/157/96","source":{"id":"https://openalex.org/S4210234078","display_name":"STATISTICS COMPUTING AND INTERDISCIPLINARY RESEARCH","issn_l":"2707-7101","issn":["2707-7101","2788-7464"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4210126226","host_organization_name":"The Women University Multan","host_organization_lineage":["https://openalex.org/I4210126226"],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"STATISTICS, COMPUTING AND INTERDISCIPLINARY RESEARCH","raw_type":"journal-article"},"parent_works":[],"lineage":["https://openalex.org/W4403730655"],"child_works_count":0,"type":"review","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://scir.wum.edu.pk/index.php/ojs/article/download/157/96","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5106866242","display_name":"Aiza Shabir","orcid":"https://orcid.org/0009-0003-2433-1621","observed_orcids":["https://orcid.org/0009-0003-2433-1621"]},"institutions":[{"id":"https://openalex.org/I4210126226","display_name":"The Women University Multan","ror":"https://ror.org/035ggvj17","country_code":"PK","type":"education","lineage":["https://openalex.org/I4210126226"]}],"countries":["PK"],"is_corresponding":false,"raw_author_name":"Aiza Shabir","raw_affiliation_strings":["Institute of Computer Science and Information Technology, The Women University Multan, Multan 60000, Pakistan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Computer Science and Information Technology, The Women University Multan, Multan 60000, Pakistan","institution_ids":["https://openalex.org/I4210126226"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106866242","display_name":"Aiza Shabir","orcid":"https://orcid.org/0009-0003-2433-1621","observed_orcids":["https://orcid.org/0009-0003-2433-1621"]},"institutions":[{"id":"https://openalex.org/I4210126226","display_name":"The Women University Multan","ror":"https://ror.org/035ggvj17","country_code":"PK","type":"education","lineage":["https://openalex.org/I4210126226"]}],"countries":["PK"],"is_corresponding":false,"raw_author_name":"Aiza Shabir","raw_affiliation_strings":["Institute of Computer Science and Information Technology, The Women University Multan, Multan 60000, Pakistan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Computer Science and Information Technology, The Women University Multan, Multan 60000, Pakistan","institution_ids":["https://openalex.org/I4210126226"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102901403","display_name":"Khawaja Tehseen Ahmed","orcid":"https://orcid.org/0000-0003-1394-4214","observed_orcids":["https://orcid.org/0000-0003-1394-4214"]},"institutions":[{"id":"https://openalex.org/I127670440","display_name":"Bahauddin Zakariya University","ror":"https://ror.org/05x817c41","country_code":"PK","type":"education","lineage":["https://openalex.org/I127670440"]}],"countries":["PK"],"is_corresponding":false,"raw_author_name":"Khawaja Tehseen Ahmed","raw_affiliation_strings":["Department of Computer Science, Bahauddin Zakariya University Multan, Pakistan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Bahauddin Zakariya University Multan, Pakistan","institution_ids":["https://openalex.org/I127670440"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102901403","display_name":"Khawaja Tehseen Ahmed","orcid":"https://orcid.org/0000-0003-1394-4214","observed_orcids":["https://orcid.org/0000-0003-1394-4214"]},"institutions":[{"id":"https://openalex.org/I127670440","display_name":"Bahauddin Zakariya University","ror":"https://ror.org/05x817c41","country_code":"PK","type":"education","lineage":["https://openalex.org/I127670440"]}],"countries":["PK"],"is_corresponding":false,"raw_author_name":"Khawaja Tehseen Ahmed","raw_affiliation_strings":["Department of Computer Science, Bahauddin Zakariya University Multan, Pakistan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Bahauddin Zakariya University Multan, Pakistan","institution_ids":["https://openalex.org/I127670440"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070702816","display_name":"Khadija Kanwal","orcid":"https://orcid.org/0009-0001-2300-9636","observed_orcids":["https://orcid.org/0009-0001-2300-9636"]},"institutions":[{"id":"https://openalex.org/I4210126226","display_name":"The Women University Multan","ror":"https://ror.org/035ggvj17","country_code":"PK","type":"education","lineage":["https://openalex.org/I4210126226"]}],"countries":["PK"],"is_corresponding":false,"raw_author_name":"Khadija Kanwal","raw_affiliation_strings":["Institute of Computer Science and Information Technology, The Women University Multan, Multan 60000, Pakistan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Computer Science and Information Technology, The Women University Multan, Multan 60000, Pakistan","institution_ids":["https://openalex.org/I4210126226"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005083795","display_name":"Afshan Almas","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210126226","display_name":"The Women University Multan","ror":"https://ror.org/035ggvj17","country_code":"PK","type":"education","lineage":["https://openalex.org/I4210126226"]}],"countries":["PK"],"is_corresponding":false,"raw_author_name":"Afshan Almas","raw_affiliation_strings":["Institute of Computer Science and Information Technology, The Women University Multan, Multan 60000, Pakistan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Computer Science and Information Technology, The Women University Multan, Multan 60000, Pakistan","institution_ids":["https://openalex.org/I4210126226"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040370658","display_name":"Sehrish Raza","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210126226","display_name":"The Women University Multan","ror":"https://ror.org/035ggvj17","country_code":"PK","type":"education","lineage":["https://openalex.org/I4210126226"]}],"countries":["PK"],"is_corresponding":false,"raw_author_name":"Sehrish Raza","raw_affiliation_strings":["Institute of Computer Science and Information Technology, The Women University Multan, Multan 60000, Pakistan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Computer Science and Information Technology, The Women University Multan, Multan 60000, Pakistan","institution_ids":["https://openalex.org/I4210126226"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070346548","display_name":"Meherwar Fatima","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210126226","display_name":"The Women University Multan","ror":"https://ror.org/035ggvj17","country_code":"PK","type":"education","lineage":["https://openalex.org/I4210126226"]}],"countries":["PK"],"is_corresponding":false,"raw_author_name":"Meherwar Fatima","raw_affiliation_strings":["Institute of Computer Science and Information Technology, The Women University Multan, Multan 60000, Pakistan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Computer Science and Information Technology, The Women University Multan, Multan 60000, Pakistan","institution_ids":["https://openalex.org/I4210126226"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5007292153","display_name":"Tahir Abbas","orcid":"https://orcid.org/0000-0002-0968-3261","observed_orcids":["https://orcid.org/0000-0002-0968-3261"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tahir Abbas","raw_affiliation_strings":["Department of Computer Science, TIMES Institute, Multan, 60000, Pakistan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, TIMES Institute, Multan, 60000, Pakistan","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0947,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":{"value":0.22159549,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":96},"biblio":{"volume":"6","issue":"1","first_page":"33","last_page":"56"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9434000253677368,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9434000253677368,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.04560000076889992,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.0017999999690800905,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/attention-mechanism","display_name":"attention mechanism","score":0.9779999852180481},{"id":"https://openalex.org/keywords/attention-based-neural-networks","display_name":"attention-based neural networks","score":0.7260000109672546},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"natural language processing","score":0.8130000233650208},{"id":"https://openalex.org/keywords/question-answering","display_name":"question answering","score":0.8560000061988831},{"id":"https://openalex.org/keywords/machine-translation","display_name":"machine translation","score":0.8320000171661377},{"id":"https://openalex.org/keywords/sentiment-analysis","display_name":"sentiment analysis","score":0.8389999866485596},{"id":"https://openalex.org/keywords/semantic-analysis","display_name":"semantic analysis","score":0.9559999704360962},{"id":"https://openalex.org/keywords/recurrent-neural-networks","display_name":"recurrent neural networks","score":0.7820000052452087},{"id":"https://openalex.org/keywords/performance-degradation","display_name":"performance degradation","score":0.7089999914169312},{"id":"https://openalex.org/keywords/interpretability","display_name":"interpretability","score":0.8379999995231628},{"id":"https://openalex.org/keywords/model-architecture","display_name":"model architecture","score":0.33899998664855957}],"concepts":[{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.519332766532898},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.49117013812065125},{"id":"https://openalex.org/C2776608160","wikidata":"https://www.wikidata.org/wiki/Q4785462","display_name":"Natural (archaeology)","level":2,"score":0.46852701902389526},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.431674987077713},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.321475625038147},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.2317672073841095},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.12101715803146362},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.52700/scir.v6i1.157","doi":"https://doi.org/10.52700/scir.v6i1.157","is_oa":true,"landing_page_url":"https://doi.org/10.52700/scir.v6i1.157","pdf_url":"https://scir.wum.edu.pk/index.php/ojs/article/download/157/96","source":{"id":"https://openalex.org/S4210234078","display_name":"STATISTICS COMPUTING AND INTERDISCIPLINARY RESEARCH","issn_l":"2707-7101","issn":["2707-7101","2788-7464"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4210126226","host_organization_name":"The Women University Multan","host_organization_lineage":["https://openalex.org/I4210126226"],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"STATISTICS, COMPUTING AND INTERDISCIPLINARY RESEARCH","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.52700/scir.v6i1.157","doi":"https://doi.org/10.52700/scir.v6i1.157","is_oa":true,"landing_page_url":"https://doi.org/10.52700/scir.v6i1.157","pdf_url":"https://scir.wum.edu.pk/index.php/ojs/article/download/157/96","source":{"id":"https://openalex.org/S4210234078","display_name":"STATISTICS COMPUTING AND INTERDISCIPLINARY RESEARCH","issn_l":"2707-7101","issn":["2707-7101","2788-7464"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4210126226","host_organization_name":"The Women University Multan","host_organization_lineage":["https://openalex.org/I4210126226"],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"STATISTICS, COMPUTING AND INTERDISCIPLINARY RESEARCH","raw_type":"journal-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4403730655.pdf","grobid_xml":"https://content.openalex.org/works/W4403730655.grobid-xml"},"referenced_works_count":40,"referenced_works":["https://openalex.org/W854541894","https://openalex.org/W1664573881","https://openalex.org/W1793121960","https://openalex.org/W2118463056","https://openalex.org/W2133564696","https://openalex.org/W2173361515","https://openalex.org/W2201092681","https://openalex.org/W2415204069","https://openalex.org/W2415755012","https://openalex.org/W2463267937","https://openalex.org/W2463565445","https://openalex.org/W2534200568","https://openalex.org/W2626792426","https://openalex.org/W2777929561","https://openalex.org/W2793978524","https://openalex.org/W2885396331","https://openalex.org/W2891635683","https://openalex.org/W2941531368","https://openalex.org/W2948140294","https://openalex.org/W2949335953","https://openalex.org/W2949765370","https://openalex.org/W2950399211","https://openalex.org/W2950893369","https://openalex.org/W2951025380","https://openalex.org/W2952111265","https://openalex.org/W2962901607","https://openalex.org/W2963084773","https://openalex.org/W2963122061","https://openalex.org/W2963168371","https://openalex.org/W2963208801","https://openalex.org/W2963680249","https://openalex.org/W2963699608","https://openalex.org/W2963794306","https://openalex.org/W2963929190","https://openalex.org/W2964164368","https://openalex.org/W2964165804","https://openalex.org/W2964199361","https://openalex.org/W3031696893","https://openalex.org/W3092785544","https://openalex.org/W3198410552"],"related_works":["https://openalex.org/W4409078899","https://openalex.org/W2941531368","https://openalex.org/W3031696893","https://openalex.org/W4410938983","https://openalex.org/W2969262604","https://openalex.org/W4416940883","https://openalex.org/W3207964656","https://openalex.org/W4401359468","https://openalex.org/W4400087319"],"abstract_inverted_index":{"The":[0],"role":[1],"of":[2,19,81,87,115,143,166,174,185,212],"attention":[3,88,176],"models":[4,25,177],"in":[5,14,57,150,183,209],"neural":[6,54],"networks":[7,55],"has":[8,83],"drawn":[9],"much":[10],"focus":[11],"among":[12],"scholars":[13],"recent":[15],"years":[16],"mainly":[17],"because":[18],"its":[20,189,194],"efficiency":[21],"and":[22,41,59,77,96,118,121,157,188],"versatility.":[23],"Attention":[24],"are":[26,169],"important":[27],"for":[28,111,171,178],"many":[29],"Natural":[30],"Language":[31],"Processing":[32],"applications":[33],"like":[34,62,101],"question":[35],"answering,":[36],"semantic":[37],"analysis,":[38],"sentiment":[39],"analysis":[40,98],"machine":[42],"translation":[43],"performing":[44],"significantly":[45],"better":[46],"than":[47],"other":[48,91],"usual":[49],"approaches.":[50],"These":[51],"mechanisms":[52,89],"improve":[53],"capability":[56],"interpretation":[58],"tackle":[60],"issues":[61,182],"performance":[63,187],"decay,":[64],"which":[65],"affects":[66],"the":[67,85,128,132,155,167,172,175,210],"Recurrent":[68],"Neural":[69],"Networks":[70],"by":[71],"concentrating":[72],"on":[73,147],"significant":[74],"data":[75],"inputs":[76],"context.":[78],"A":[79],"stream":[80],"research":[82,114,138,168,203],"explored":[84],"integration":[86],"to":[90,130,200,204],"fields,":[92],"namely":[93],"computer":[94],"vision":[95],"graph":[97],"improving":[99],"tasks":[100],"object":[102],"detection,":[103],"image":[104],"captioning":[105],"or":[106],"node":[107],"representation":[108],"learning.":[109],"However,":[110],"more":[112,133,205],"detailed":[113],"modern":[116],"achievements":[117],"various":[119],"domains":[120],"model":[122,149,186],"architecture":[123],"structures":[124],"there":[125],"is":[126,198],"still":[127],"necessity":[129],"use":[131],"profound":[134],"survey-based":[135],"investigation.":[136],"This":[137,196],"article":[139],"provides":[140,158],"a":[141],"review":[142,197],"previous":[144],"work":[145],"done":[146],"attention-based":[148,207],"NLP.":[151,213],"It":[152],"also":[153],"outlines":[154],"problems":[156],"relevant":[159],"solutions":[160,208],"using":[161],"different":[162],"methodologies.":[163],"Future":[164],"directions":[165],"provided":[170],"improvement":[173],"NLP,":[179],"discussing":[180],"potential":[181],"terms":[184],"capacity,":[190],"as":[191,193],"well":[192],"interpretability.":[195],"intended":[199],"help":[201],"further":[202],"universal":[206],"domain":[211]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":3}],"updated_date":"2026-11-14T11:04:58.165337","created_date":"2024-10-25T00:00:00"}