{"id":"https://openalex.org/W4396861092","doi":"https://doi.org/10.1145/3664812","title":"<tt>LLMEffiChecker</tt> : Understanding and Testing Efficiency Degradation of Large Language Models","display_name":"<tt>LLMEffiChecker</tt> : Understanding and Testing Efficiency Degradation of Large Language Models","publication_year":2024,"publication_date":"2024-05-13","ids":{"openalex":"https://openalex.org/W4396861092","doi":"https://doi.org/10.1145/3664812"},"language":"en","primary_location":{"id":"doi:10.1145/3664812","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3664812","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3664812","source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3664812","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5084333121","display_name":"X.L. Feng","orcid":"https://orcid.org/0009-0001-6344-2477"},"institutions":[{"id":"https://openalex.org/I9086337","display_name":"Taiyuan University of Technology","ror":"https://ror.org/03kv08d37","country_code":"CN","type":"education","lineage":["https://openalex.org/I9086337"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Xiaoning Feng","raw_affiliation_strings":["College of Computer Science and Technology (College of Data Science), Taiyuan University of Technology, Taiyuan, China"],"affiliations":[{"raw_affiliation_string":"College of Computer Science and Technology (College of Data Science), Taiyuan University of Technology, Taiyuan, China","institution_ids":["https://openalex.org/I9086337"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031631007","display_name":"Xiaohong Han","orcid":"https://orcid.org/0000-0002-8779-6528"},"institutions":[{"id":"https://openalex.org/I9086337","display_name":"Taiyuan University of Technology","ror":"https://ror.org/03kv08d37","country_code":"CN","type":"education","lineage":["https://openalex.org/I9086337"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaohong Han","raw_affiliation_strings":["College of Computer Science and Technology (College of Data Science), Taiyuan University of Technology, Taiyuan, China"],"affiliations":[{"raw_affiliation_string":"College of Computer Science and Technology (College of Data Science), Taiyuan University of Technology, Taiyuan, China","institution_ids":["https://openalex.org/I9086337"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045987094","display_name":"S. H. Chen","orcid":"https://orcid.org/0000-0001-5035-3398"},"institutions":[{"id":"https://openalex.org/I162577319","display_name":"The University of Texas at Dallas","ror":"https://ror.org/049emcs32","country_code":"US","type":"education","lineage":["https://openalex.org/I162577319"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Simin Chen","raw_affiliation_strings":["Computer Science, The University of Texas at Dallas, Richardson, United States"],"affiliations":[{"raw_affiliation_string":"Computer Science, The University of Texas at Dallas, Richardson, United States","institution_ids":["https://openalex.org/I162577319"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100613524","display_name":"Wei Yang","orcid":"https://orcid.org/0000-0002-5338-7347"},"institutions":[{"id":"https://openalex.org/I162577319","display_name":"The University of Texas at Dallas","ror":"https://ror.org/049emcs32","country_code":"US","type":"education","lineage":["https://openalex.org/I162577319"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wei Yang","raw_affiliation_strings":["University of Texas at Dallas, Richardson, United States"],"affiliations":[{"raw_affiliation_string":"University of Texas at Dallas, Richardson, United States","institution_ids":["https://openalex.org/I162577319"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5084333121"],"corresponding_institution_ids":["https://openalex.org/I9086337"],"apc_list":null,"apc_paid":null,"fwci":4.0007,"has_fulltext":false,"cited_by_count":11,"citation_normalized_percentile":{"value":0.94292316,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":"33","issue":"7","first_page":"1","last_page":"38"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.98089998960495,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9728999733924866,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7619261741638184},{"id":"https://openalex.org/keywords/degradation","display_name":"Degradation (telecommunications)","score":0.5194570422172546}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7619261741638184},{"id":"https://openalex.org/C2779679103","wikidata":"https://www.wikidata.org/wiki/Q5251805","display_name":"Degradation (telecommunications)","level":2,"score":0.5194570422172546},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3664812","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3664812","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3664812","source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3664812","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3664812","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3664812","source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.4000000059604645,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4396861092.pdf"},"referenced_works_count":79,"referenced_works":["https://openalex.org/W148400104","https://openalex.org/W2163931728","https://openalex.org/W2487770199","https://openalex.org/W2562731582","https://openalex.org/W2607219512","https://openalex.org/W2612445135","https://openalex.org/W2747329762","https://openalex.org/W2799194071","https://openalex.org/W2889580470","https://openalex.org/W2905526464","https://openalex.org/W2946609015","https://openalex.org/W2949128310","https://openalex.org/W2954629067","https://openalex.org/W2963125010","https://openalex.org/W2963327228","https://openalex.org/W2963393494","https://openalex.org/W2963857521","https://openalex.org/W2963859254","https://openalex.org/W2964164993","https://openalex.org/W2970295111","https://openalex.org/W2979826702","https://openalex.org/W2981852735","https://openalex.org/W2982138961","https://openalex.org/W2988194011","https://openalex.org/W2996851481","https://openalex.org/W2998277219","https://openalex.org/W3001434439","https://openalex.org/W3034397670","https://openalex.org/W3034866909","https://openalex.org/W3035736465","https://openalex.org/W3090623329","https://openalex.org/W3090988182","https://openalex.org/W3091407209","https://openalex.org/W3091441586","https://openalex.org/W3100726446","https://openalex.org/W3101449015","https://openalex.org/W3102720581","https://openalex.org/W3135132088","https://openalex.org/W3160537436","https://openalex.org/W3163644064","https://openalex.org/W3175663471","https://openalex.org/W3176617251","https://openalex.org/W3177813494","https://openalex.org/W3193977407","https://openalex.org/W3198685994","https://openalex.org/W3215698779","https://openalex.org/W4225591000","https://openalex.org/W4226227340","https://openalex.org/W4229005866","https://openalex.org/W4287024925","https://openalex.org/W4288089799","https://openalex.org/W4293584023","https://openalex.org/W4297775537","https://openalex.org/W4307079201","https://openalex.org/W4308643054","https://openalex.org/W4308760226","https://openalex.org/W4308902180","https://openalex.org/W4309088836","https://openalex.org/W4311642023","https://openalex.org/W4312648739","https://openalex.org/W4313563684","https://openalex.org/W4321153003","https://openalex.org/W4322718191","https://openalex.org/W4365512576","https://openalex.org/W4366204357","https://openalex.org/W4384200891","https://openalex.org/W4384345748","https://openalex.org/W4384918448","https://openalex.org/W4385562549","https://openalex.org/W4385571706","https://openalex.org/W4385988359","https://openalex.org/W4386075644","https://openalex.org/W4386185625","https://openalex.org/W4391136507","https://openalex.org/W6734483310","https://openalex.org/W6798182279","https://openalex.org/W6800166007","https://openalex.org/W6811340617","https://openalex.org/W6854692045"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052","https://openalex.org/W2382290278","https://openalex.org/W4395014643"],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"have":[4,169],"received":[5,48],"much":[6],"recent":[7],"attention":[8],"due":[9,37],"to":[10,38,59,96,153,170,175,226,238,255,313,317],"their":[11],"human-level":[12],"accuracy.":[13],"While":[14],"existing":[15],"works":[16],"mostly":[17],"focus":[18],"on":[19,124,273,303],"either":[20],"improving":[21],"accuracy":[22,25],"or":[23,325],"testing":[24],"robustness,":[26],"the":[27,56,72,106,110,117,120,136,161,177,195,217,241,248,257,263,328,342],"computation":[28,64,98,111],"efficiency":[29,65,99,112],"of":[30,34,77,113,116,139,145,163,235,250,265],"LLMs,":[31,80],"which":[32,184],"is":[33,104,152],"paramount":[35],"importance":[36],"often":[39,128],"vast":[40],"generation":[41,162],"demands":[42],"and":[43,61,75,141,191,208,214,230,243,293,308,315],"real-time":[44],"requirements,":[45],"has":[46],"surprisingly":[47],"little":[49],"attention.":[50],"In":[51,194,216],"this":[52],"article,":[53],"we":[54,81,268],"make":[55],"first":[57],"attempt":[58],"understand":[60],"test":[62,155],"potential":[63],"robustness":[66],"in":[67,86,92,327,345],"state-of-the-art":[68],"LLMs.":[69],"By":[70],"analyzing":[71],"working":[73],"mechanism":[74],"implementation":[76],"20,543":[78],"public-accessible":[79],"observe":[82],"a":[83,142,200,206,222,270],"fundamental":[84],"property":[85],"LLMs":[87,114,167],"that":[88,105,157,166,203,299,335],"could":[89,158],"be":[90],"manipulated":[91],"an":[93,127],"adversarial":[94],"manner":[95],"reduce":[97],"significantly.":[100],"Our":[101,149,331],"interesting":[102],"observation":[103],"output":[107,121],"length":[108,122],"determines":[109],"instead":[115],"input,":[118],"where":[119],"depends":[123],"two":[125],"factors:":[126],"sufficiently":[129,159],"large":[130],"yet":[131],"pessimistic":[132],"pre-configured":[133,178],"threshold":[134],"controlling":[135],"max":[137],"number":[138],"iterations":[140,174],"runtime-generated":[143],"end":[144],"sentence":[146],"(EOS)":[147],"token.":[148],"key":[150],"motivation":[151],"generate":[154],"inputs":[156,254,336],"delay":[160,247],"EOS":[164],"such":[165],"would":[168],"go":[171],"through":[172],"enough":[173],"satisfy":[176],"threshold.":[179,260],"We":[180],"present":[181],"LLMEffiChecker":[182,198,220,266,300,339],",":[183,267],"can":[185,301],"work":[186],"under":[187],"both":[188],"white-box":[189,196,242],"setting":[190],"black-box":[192,218,244],"setting.":[193],"scenario,":[197,219],"develops":[199],"gradient-guided":[201],"technique":[202],"searches":[204],"for":[205],"minimal":[207],"unnoticeable":[209],"perturbation":[210,237],"at":[211],"character-level,":[212],"token-level,":[213],"structure-level.":[215],"employs":[221],"causal":[223],"inference-based":[224],"approach":[225],"find":[227],"critical":[228],"tokens":[229],"similarly":[231],"applies":[232],"three":[233],"levels":[234],"imperceptible":[236],"them.":[239],"Both":[240],"settings":[245],"effectively":[246],"appearance":[249],"EOS,":[251],"compelling":[252],"these":[253],"reach":[256],"naturally":[258],"unreachable":[259],"To":[261],"demonstrate":[262],"effectiveness":[264],"conduct":[269],"systematic":[271],"evaluation":[272],"nine":[274],"publicly":[275],"available":[276],"LLMs:":[277],"Google":[278,289],"T5,":[279],"AllenAI":[280],"WMT14,":[281],"Helsinki-NLP":[282],"translator,":[283,287],"Facebook":[284],"FairSeq,":[285],"UNICAMP-DL":[286],"MarianMT,":[288],"FLAN-T5,":[290],"MBZUAI":[291],"LaMini-GPT,":[292],"Salesforce":[294],"CodeGen.":[295],"Experimental":[296],"results":[297],"show":[298],"increase":[302],"average":[304],"LLMs\u2019":[305],"response":[306],"latency":[307],"energy":[309],"consumption":[310],"by":[311,320,338],"325%":[312],"3,244%":[314],"344%":[316],"3,616%,":[318],"respectively,":[319],"perturbing":[321],"just":[322],"one":[323],"character":[324],"token":[326],"input":[329],"sentence.":[330],"case":[332],"study":[333],"shows":[334],"generated":[337],"significantly":[340],"affect":[341],"battery":[343,355],"power":[344,356],"real-world":[346],"mobile":[347],"devices":[348],"(i.e.,":[349],"drain":[350],"more":[351],"than":[352,357],"30":[353],"times":[354],"normal":[358],"inputs).":[359]},"counts_by_year":[{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":3}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2024-05-14T00:00:00"}