{"id":"https://openalex.org/W6892249118","doi":"https://doi.org/10.5075/epfl-thesis-8607","title":"Stop Wasting my FLOPS: Improving the Efficiency of Deep Learning Models","display_name":"Stop Wasting my FLOPS: Improving the Efficiency of Deep Learning Models","publication_year":2022,"publication_date":"2022-01-01","ids":{"openalex":"https://openalex.org/W6892249118","doi":"https://doi.org/10.5075/epfl-thesis-8607"},"language":"en","primary_location":{"id":"pmh:oai:infoscience.epfl.ch:294302","is_oa":true,"landing_page_url":"https://infoscience.epfl.ch/handle/20.500.14299/188206","pdf_url":null,"source":{"id":"https://openalex.org/S4306400487","display_name":"Infoscience (Ecole Polytechnique F\u00e9d\u00e9rale de Lausanne)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"doctoral thesis"},"type":"dissertation","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://infoscience.epfl.ch/handle/20.500.14299/188206","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Katharopoulos, Angelos","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Katharopoulos, Angelos","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T12859","display_name":"Cell Image Analysis Techniques","score":0.1542000025510788,"subfield":{"id":"https://openalex.org/subfields/1304","display_name":"Biophysics"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T12859","display_name":"Cell Image Analysis Techniques","score":0.1542000025510788,"subfield":{"id":"https://openalex.org/subfields/1304","display_name":"Biophysics"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.13179999589920044,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.09399999678134918,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.6723999977111816},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.6553000211715698},{"id":"https://openalex.org/keywords/autoregressive-model","display_name":"Autoregressive model","score":0.6388000249862671},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.49309998750686646},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4424999952316284},{"id":"https://openalex.org/keywords/computational-complexity-theory","display_name":"Computational complexity theory","score":0.3952000141143799},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.3824999928474426},{"id":"https://openalex.org/keywords/sampling","display_name":"Sampling (signal processing)","score":0.3637999892234802},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.3578000068664551}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7670999765396118},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.7408999800682068},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.6723999977111816},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.6553000211715698},{"id":"https://openalex.org/C159877910","wikidata":"https://www.wikidata.org/wiki/Q2202883","display_name":"Autoregressive model","level":2,"score":0.6388000249862671},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6190999746322632},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.49309998750686646},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4424999952316284},{"id":"https://openalex.org/C179799912","wikidata":"https://www.wikidata.org/wiki/Q205084","display_name":"Computational complexity theory","level":2,"score":0.3952000141143799},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.3824999928474426},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.3637999892234802},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.3578000068664551},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.34360000491142273},{"id":"https://openalex.org/C81363708","wikidata":"https://www.wikidata.org/wiki/Q17084460","display_name":"Convolutional neural network","level":2,"score":0.33889999985694885},{"id":"https://openalex.org/C174348530","wikidata":"https://www.wikidata.org/wiki/Q188635","display_name":"Bridging (networking)","level":2,"score":0.3260999917984009},{"id":"https://openalex.org/C3832189","wikidata":"https://www.wikidata.org/wiki/Q8588916","display_name":"Models of neural computation","level":3,"score":0.3165999948978424},{"id":"https://openalex.org/C147168706","wikidata":"https://www.wikidata.org/wiki/Q1457734","display_name":"Recurrent neural network","level":3,"score":0.3093000054359436},{"id":"https://openalex.org/C52740198","wikidata":"https://www.wikidata.org/wiki/Q1539564","display_name":"Importance sampling","level":3,"score":0.30790001153945923},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.29510000348091125},{"id":"https://openalex.org/C66024118","wikidata":"https://www.wikidata.org/wiki/Q1122506","display_name":"Computational model","level":2,"score":0.2915000021457672},{"id":"https://openalex.org/C2778869765","wikidata":"https://www.wikidata.org/wiki/Q6028363","display_name":"Inefficiency","level":2,"score":0.28540000319480896},{"id":"https://openalex.org/C127964446","wikidata":"https://www.wikidata.org/wiki/Q1092142","display_name":"Computational resource","level":3,"score":0.2694000005722046},{"id":"https://openalex.org/C151406439","wikidata":"https://www.wikidata.org/wiki/Q186588","display_name":"Time series","level":2,"score":0.2660999894142151},{"id":"https://openalex.org/C150899416","wikidata":"https://www.wikidata.org/wiki/Q1820378","display_name":"Transfer of learning","level":2,"score":0.26460000872612},{"id":"https://openalex.org/C129844170","wikidata":"https://www.wikidata.org/wiki/Q41299","display_name":"Quadratic equation","level":2,"score":0.2506999969482422}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:infoscience.epfl.ch:294302","is_oa":true,"landing_page_url":"https://infoscience.epfl.ch/handle/20.500.14299/188206","pdf_url":null,"source":{"id":"https://openalex.org/S4306400487","display_name":"Infoscience (Ecole Polytechnique F\u00e9d\u00e9rale de Lausanne)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"doctoral thesis"},{"id":"doi:10.5075/epfl-thesis-8607","is_oa":true,"landing_page_url":"https://doi.org/10.5075/epfl-thesis-8607","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Dissertation"}],"best_oa_location":{"id":"pmh:oai:infoscience.epfl.ch:294302","is_oa":true,"landing_page_url":"https://infoscience.epfl.ch/handle/20.500.14299/188206","pdf_url":null,"source":{"id":"https://openalex.org/S4306400487","display_name":"Infoscience (Ecole Polytechnique F\u00e9d\u00e9rale de Lausanne)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"doctoral thesis"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","score":0.44190531969070435,"display_name":"Industry, innovation and infrastructure"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Deep":[0],"neural":[1,49,58,69,111,234],"networks":[2,112,235],"have":[3,97],"completely":[4],"revolutionized":[5],"the":[6,45,54,93,107,140,166,179,182,220,227,273,293],"field":[7],"of":[8,47,57,109,139,145,165,246,272],"machine":[9],"learning":[10],"by":[11,30,151,269],"achieving":[12],"state-of-the-art":[13],"results":[14],"on":[15,78],"various":[16,110],"tasks":[17],"ranging":[18],"from":[19,153],"computer":[20],"vision":[21],"to":[22,75,83,115,160,200,215,219,243,292],"protein":[23],"folding.":[24],"However,":[25],"their":[26,31],"application":[27],"is":[28,129,267],"hindered":[29],"large":[32,133],"computational":[33,121,141],"and":[34,91,142,181,203,232,236,288,312],"memory":[35,143],"requirements.":[36],"In":[37,297],"this":[38,150],"thesis,":[39],"we":[40,52,124,196,225,252,299],"propose":[41],"methods":[42],"for":[43,67,87,131,193,208],"improving":[44],"efficiency":[46],"deep":[48,68],"networks.":[50,70],"Firstly,":[51],"tackle":[53],"sample":[55],"inefficiency":[56],"network":[59],"training":[60,88],"with":[61,136,217,263,313],"an":[62,185],"importance":[63],"sampling":[64,117,152],"algorithm":[65,72,104],"suitable":[66,130],"This":[71,266],"allows":[73],"us":[74],"focus":[76],"computation":[77,289],"datapoints":[79],"that":[80,95,102,128,173,210,238,258,279,301],"are":[81],"going":[82],"provide":[84],"useful":[85],"gradients":[86],"our":[89,103,174,198,239,280],"models":[90,308],"ignore":[92],"ones":[94],"will":[96],"negligible":[98],"gradients.":[99],"We":[100,148,171,277],"show":[101,237],"can":[105,176,259,304],"improve":[106],"performance":[108,287],"when":[113],"compared":[114],"uniform":[116],"under":[118],"a":[119,126,137,154,163,205,256,283],"fixed":[120],"budget.":[122],"Secondly,":[123],"design":[125],"model":[127,175],"processing":[132],"input":[134,167,221,274],"images":[135],"fraction":[138],"requirements":[144],"traditional":[146],"approaches.":[147],"achieve":[149],"data-dependent":[155],"attention":[156,180,199,255,303],"distribution":[157],"in":[158,168,184,290,316],"order":[159],"only":[161,189],"process":[162],"portion":[164],"high":[169],"resolution.":[170],"demonstrate":[172,300],"learn":[177],"both":[178],"features":[183],"end-to-end":[186],"fashion":[187],"using":[188,275],"single":[190],"image-wise":[191],"labels":[192],"supervision.":[194],"Subsequently,":[195],"shift":[197],"transformer":[201,295,307],"architectures":[202],"introduce":[204],"kernelized":[206],"formulation":[207,240,281],"self-attention":[209],"reduces":[211],"its":[212],"quadratic":[213],"complexity":[214],"linear":[216],"respect":[218],"sequence's":[222],"length.":[223],"Furthermore,":[224],"uncover":[226],"relationship":[228],"between":[229,286],"autoregressive":[230,249],"transformers":[231,262],"recurrent":[233],"enables":[241],"up":[242],"3":[244],"orders":[245],"magnitude":[247],"faster":[248],"inference.":[250],"Finally,":[251],"develop":[253],"clustered,":[254],"method":[257],"approximate":[260,305],"softmax":[261],"reduced":[264],"computation.":[265],"achieved":[268],"grouping":[270],"elements":[271],"clustering.":[276],"showcase":[278],"provides":[282],"better":[284],"trade-off":[285],"comparison":[291],"original":[294],"architecture.":[296],"addition,":[298],"clustered":[302],"pretrained":[306],"without":[309],"any":[310],"fine-tuning":[311],"minimal":[314],"loss":[315],"performance.":[317]},"counts_by_year":[],"updated_date":"2026-07-23T05:56:39.545243","created_date":"2025-10-10T00:00:00"}