{"id":"https://openalex.org/W7028512258","doi":null,"title":"Exploring Causal Attention Models in Transformers for Large Language Models","display_name":"Exploring Causal Attention Models in Transformers for Large Language Models","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W7028512258"},"language":"pt","primary_location":{"id":"pmh:oai:repositorio.ulisboa.pt:10400.5/96896","doi":null,"is_oa":true,"landing_page_url":"http://hdl.handle.net/10400.5/96896","pdf_url":"https://repositorio.ulisboa.pt/bitstreams/6389e837-2979-4d2c-b41a-55df8399ff5c/download","source":{"id":"https://openalex.org/S4306402433","display_name":"Portuguese National Funding Agency for Science, Research and Technology (RCAAP Project by FCT)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"master thesis"},"volume":null,"type":"dissertation","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://repositorio.ulisboa.pt/bitstreams/6389e837-2979-4d2c-b41a-55df8399ff5c/download","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Terroa, Jo\u00e3o Filipe Gon\u00e7alves Vieira","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Terroa, Jo\u00e3o Filipe Gon\u00e7alves Vieira","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9351000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9351000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.0494999997317791,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.006000000052154064,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/self-attention","display_name":"self-attention","score":0.8029999732971191},{"id":"https://openalex.org/keywords/transformer","display_name":"transformer models","score":0.460999995470047},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7950000166893005},{"id":"https://openalex.org/keywords/model-efficiency","display_name":"model efficiency","score":0.531000018119812},{"id":"https://openalex.org/keywords/computational-cost-reduction","display_name":"computational complexity reduction","score":0.5550000071525574},{"id":"https://openalex.org/keywords/resource-constrained-devices","display_name":"resource-constrained devices","score":0.6200000047683716}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5598999857902527},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.49720001220703125},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4406000077724457},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38830000162124634},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.3319000005722046},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.323199987411499},{"id":"https://openalex.org/C2983448237","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Language understanding","level":2,"score":0.27000001072883606},{"id":"https://openalex.org/C11671645","wikidata":"https://www.wikidata.org/wiki/Q5054567","display_name":"Causal model","level":2,"score":0.25999999046325684},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2433999925851822},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.2363000065088272}],"mesh":[],"locations_count":1,"locations":[{"id":"pmh:oai:repositorio.ulisboa.pt:10400.5/96896","doi":null,"is_oa":true,"landing_page_url":"http://hdl.handle.net/10400.5/96896","pdf_url":"https://repositorio.ulisboa.pt/bitstreams/6389e837-2979-4d2c-b41a-55df8399ff5c/download","source":{"id":"https://openalex.org/S4306402433","display_name":"Portuguese National Funding Agency for Science, Research and Technology (RCAAP Project by FCT)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"master thesis"}],"best_oa_location":{"id":"pmh:oai:repositorio.ulisboa.pt:10400.5/96896","doi":null,"is_oa":true,"landing_page_url":"http://hdl.handle.net/10400.5/96896","pdf_url":"https://repositorio.ulisboa.pt/bitstreams/6389e837-2979-4d2c-b41a-55df8399ff5c/download","source":{"id":"https://openalex.org/S4306402433","display_name":"Portuguese National Funding Agency for Science, Research and Technology (RCAAP Project by FCT)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"master thesis"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7028512258.pdf","grobid_xml":"https://content.openalex.org/works/W7028512258.grobid-xml"},"referenced_works_count":129,"referenced_works":["https://openalex.org/W1816313093","https://openalex.org/W1821462560","https://openalex.org/W2133564696","https://openalex.org/W2267186426","https://openalex.org/W2613904329","https://openalex.org/W2626778328","https://openalex.org/W2763421725","https://openalex.org/W2767989436","https://openalex.org/W2787214294","https://openalex.org/W2789541106","https://openalex.org/W2866343820","https://openalex.org/W2885185669","https://openalex.org/W2888520903","https://openalex.org/W2894384847","https://openalex.org/W2896457183","https://openalex.org/W2897105542","https://openalex.org/W2911109671","https://openalex.org/W2912521296","https://openalex.org/W2940744433","https://openalex.org/W2947476638","https://openalex.org/W2948981900","https://openalex.org/W2949417870","https://openalex.org/W2949650786","https://openalex.org/W2949680561","https://openalex.org/W2955227499","https://openalex.org/W2962785754","https://openalex.org/W2965373594","https://openalex.org/W2968210605","https://openalex.org/W2969958763","https://openalex.org/W2970049541","https://openalex.org/W2970290563","https://openalex.org/W2970903692","https://openalex.org/W2970971581","https://openalex.org/W2971524460","https://openalex.org/W2977944219","https://openalex.org/W2978017171","https://openalex.org/W2979417040","https://openalex.org/W2981852735","https://openalex.org/W2986922898","https://openalex.org/W2988394319","https://openalex.org/W2988945824","https://openalex.org/W2989571009","https://openalex.org/W2994673210","https://openalex.org/W2996035354","https://openalex.org/W2997517014","https://openalex.org/W2998108143","https://openalex.org/W3005680577","https://openalex.org/W3006292807","https://openalex.org/W3006683367","https://openalex.org/W3006983028","https://openalex.org/W3010476030","https://openalex.org/W3015468748","https://openalex.org/W3021293129","https://openalex.org/W3023662336","https://openalex.org/W3030163527","https://openalex.org/W3033187248","https://openalex.org/W3033188311","https://openalex.org/W3033529678","https://openalex.org/W3033943443","https://openalex.org/W3034573343","https://openalex.org/W3035038672","https://openalex.org/W3037932933","https://openalex.org/W3041753721","https://openalex.org/W3045733172","https://openalex.org/W3085139254","https://openalex.org/W3091156754","https://openalex.org/W3094502228","https://openalex.org/W3112689365","https://openalex.org/W3115690768","https://openalex.org/W3118781290","https://openalex.org/W3119866685","https://openalex.org/W3122154272","https://openalex.org/W3131343002","https://openalex.org/W3161873870","https://openalex.org/W3164045210","https://openalex.org/W3166702123","https://openalex.org/W3170719901","https://openalex.org/W3170724489","https://openalex.org/W3171649327","https://openalex.org/W3173365702","https://openalex.org/W3174794493","https://openalex.org/W3176948526","https://openalex.org/W3177318507","https://openalex.org/W3196318247","https://openalex.org/W3200062921","https://openalex.org/W3207091856","https://openalex.org/W4223891676","https://openalex.org/W4224308101","https://openalex.org/W4226073878","https://openalex.org/W4226146865","https://openalex.org/W4226271314","https://openalex.org/W4293788414","https://openalex.org/W4295689449","https://openalex.org/W4304697829","https://openalex.org/W4312348652","https://openalex.org/W4322825218","https://openalex.org/W4327810158","https://openalex.org/W4377089791","https://openalex.org/W4377121468","https://openalex.org/W4378505330","https://openalex.org/W4379469208","https://openalex.org/W4383473945","https://openalex.org/W4384918448","https://openalex.org/W4387294617","https://openalex.org/W4387355950","https://openalex.org/W4387561528","https://openalex.org/W4388444691","https://openalex.org/W4388979610","https://openalex.org/W4389518760","https://openalex.org/W4389519226","https://openalex.org/W4389520033","https://openalex.org/W4389523718","https://openalex.org/W4389524067","https://openalex.org/W4389524473","https://openalex.org/W4390041933","https://openalex.org/W4390632065","https://openalex.org/W4390874069","https://openalex.org/W4392240262","https://openalex.org/W4394698525","https://openalex.org/W4402670549","https://openalex.org/W4404783458","https://openalex.org/W4411403346","https://openalex.org/W7133185274","https://openalex.org/W7133191579","https://openalex.org/W7133221612","https://openalex.org/W7133224090","https://openalex.org/W7133225605","https://openalex.org/W7133227460","https://openalex.org/W7133244816"],"related_works":["https://openalex.org/W7082274033","https://openalex.org/W6960079315","https://openalex.org/W7045431622","https://openalex.org/W7168036730","https://openalex.org/W7197046484","https://openalex.org/W7142768570","https://openalex.org/W4402683966","https://openalex.org/W7197638924","https://openalex.org/W7207058747","https://openalex.org/W7075933642"],"abstract_inverted_index":{"Transformers":[0],"have":[1],"played":[2],"a":[3,186,235],"significant":[4,14,264],"role":[5],"in":[6,16],"Natural":[7],"Language":[8],"Processing":[9],"(NLP)":[10],"since":[11],"2017,":[12],"enabling":[13],"advancements":[15],"applications":[17],"like":[18],"machine":[19],"translation":[20],"and":[21,35,52,117,122,144,152,162,210,241,257,275,277,290],"text":[22],"generation.":[23],"Despite":[24],"their":[25],"success,":[26],"they":[27],"face":[28],"challenges":[29],"such":[30],"as":[31,185],"high":[32],"computational":[33,71],"costs":[34],"environmental":[36],"impacts":[37],"due":[38],"to":[39,67,111,238,263,272,281,286],"energy":[40],"consumption.":[41],"Recent":[42],"research":[43],"has":[44],"focused":[45],"on":[46,56,61,84],"the":[47,113,159,168,175,182,202,249,288,293,296],"development":[48],"of":[49,76,295],"more":[50],"compact":[51],"efficient":[53],"models":[54,229,274],"operable":[55],"resource-limited":[57],"devices":[58],"without":[59,99,230],"reliance":[60],"cloud":[62],"infrastructures.":[63],"Optimizing":[64],"transformers":[65,77],"seeks":[66],"balance":[68],"performance":[69],"with":[70,81,197,214],"resources.":[72],"The":[73,155,171],"fundamental":[74],"structure":[75],"remains":[78],"largely":[79],"unchanged,":[80],"understanding":[82],"based":[83],"empirical":[85],"observations,":[86],"leaving":[87],"theoretical":[88],"gaps.":[89],"Studies":[90],"suggest":[91,220],"that":[92,190,221,248],"components":[93],"considered":[94],"essential":[95],"can":[96,225,261],"be":[97,255],"removed":[98],"compromising":[100],"performance,":[101],"indicating":[102],"potential":[103],"for":[104,125],"reevaluating":[105],"transformer":[106],"components.":[107],"This":[108],"study":[109,246],"aims":[110],"analyze":[112],"self-attention":[114,160,184,204,251],"mechanism":[115,161],"theoretically":[116],"empirically,":[118],"examining":[119],"existing":[120],"optimizations":[121],"evaluating":[123,178],"alternatives":[124,166],"improvements.":[126,265],"Five":[127],"modifications":[128],"were":[129],"designed:":[130],"Simple":[131,140],"Self-Attention":[132,135,138,142,147,193],"(SSA),":[133],"Layered":[134,141,146,192],"(LSA),":[136],"Variable":[137,145,191],"(VSA),":[139],"(SLSA),":[143],"(VLSA).":[148],"Implementation":[149],"involved":[150],"exploratory":[151,156],"confirmatory":[153,172],"phases.":[154],"phase":[157,173],"altered":[158],"extensively":[163],"tested":[164],"promising":[165],"using":[167],"nanoGPT":[169],"repository.":[170],"fine-tuned":[174],"best-performing":[176],"mechanisms,":[177],"modified":[179],"versions":[180],"alongside":[181],"original":[183],"baseline.":[187],"Results":[188],"indicated":[189],"(VLSA)":[194],"models,":[195],"especially":[196],"higher":[198],"k":[199],"values,":[200],"outperformed":[201],"standard":[203],"mechanism,":[205],"achieving":[206],"lower":[207],"validation":[208],"losses":[209],"improved":[211],"generalization,":[212],"even":[213],"fewer":[215],"training":[216],"iterations.":[217],"These":[218],"findings":[219],"alternative":[222,259],"attention":[223,298],"mechanisms":[224,260,271],"enhance":[226],"transformer-based":[227],"language":[228],"extensive":[231],"architectural":[232],"changes,":[233],"offering":[234],"practical":[236],"approach":[237],"improving":[239],"efficiency":[240],"accuracy.":[242],"In":[243],"conclusion,":[244],"this":[245],"demonstrates":[247],"current":[250],"implementation":[252],"may":[253],"not":[254],"optimal,":[256],"exploring":[258],"lead":[262],"Future":[266],"work":[267],"proposes":[268],"applying":[269],"these":[270],"larger":[273],"datasets":[276],"expanding":[278],"evaluation":[279],"metrics":[280],"include":[282],"broader":[283],"benchmarks,":[284],"aiming":[285],"generalize":[287],"improvements":[289],"better":[291],"understand":[292],"benefits":[294],"proposed":[297],"mechanisms.":[299]},"counts_by_year":[],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2025-10-10T00:00:00"}