{"id":"https://openalex.org/W4406858164","doi":"https://doi.org/10.1109/tse.2025.3533972","title":"On the Workflows and Smells of Leaderboard Operations (LBOps): An Exploratory Study of Foundation Model Leaderboards","display_name":"On the Workflows and Smells of Leaderboard Operations (LBOps): An Exploratory Study of Foundation Model Leaderboards","publication_year":2025,"publication_date":"2025-01-27","ids":{"openalex":"https://openalex.org/W4406858164","doi":"https://doi.org/10.1109/tse.2025.3533972"},"language":"en","primary_location":{"id":"doi:10.1109/tse.2025.3533972","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2025.3533972","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5084952629","display_name":"Zhimin Zhao","orcid":"https://orcid.org/0000-0002-0337-808X"},"institutions":[{"id":"https://openalex.org/I204722609","display_name":"Queen's University","ror":"https://ror.org/02y72wh86","country_code":"CA","type":"education","lineage":["https://openalex.org/I204722609"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Zhimin Zhao","raw_affiliation_strings":["Software Analysis and Intelligence Laboratory (SAIL), School of Computing, Queen&#x2019;s University, Kingston, ON, Canada","Software Analysis and Intelligence Lab (SAIL), School of Computing, Queen&#x2019;s University, Kingston, ON, Canada"],"raw_orcid":"https://orcid.org/0000-0002-0337-808X","affiliations":[{"raw_affiliation_string":"Software Analysis and Intelligence Laboratory (SAIL), School of Computing, Queen&#x2019;s University, Kingston, ON, Canada","institution_ids":["https://openalex.org/I204722609"]},{"raw_affiliation_string":"Software Analysis and Intelligence Lab (SAIL), School of Computing, Queen&#x2019;s University, Kingston, ON, Canada","institution_ids":["https://openalex.org/I204722609"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063403084","display_name":"Abdul Ali Bangash","orcid":"https://orcid.org/0000-0002-5311-6061"},"institutions":[{"id":"https://openalex.org/I204722609","display_name":"Queen's University","ror":"https://ror.org/02y72wh86","country_code":"CA","type":"education","lineage":["https://openalex.org/I204722609"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Abdul Ali Bangash","raw_affiliation_strings":["Software Analysis and Intelligence Laboratory (SAIL), School of Computing, Queen&#x2019;s University, Kingston, ON, Canada","Software Analysis and Intelligence Lab (SAIL), School of Computing, Queen&#x2019;s University, Kingston, ON, Canada"],"raw_orcid":"https://orcid.org/0000-0002-5311-6061","affiliations":[{"raw_affiliation_string":"Software Analysis and Intelligence Laboratory (SAIL), School of Computing, Queen&#x2019;s University, Kingston, ON, Canada","institution_ids":["https://openalex.org/I204722609"]},{"raw_affiliation_string":"Software Analysis and Intelligence Lab (SAIL), School of Computing, Queen&#x2019;s University, Kingston, ON, Canada","institution_ids":["https://openalex.org/I204722609"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032675988","display_name":"Filipe R. Cogo","orcid":"https://orcid.org/0000-0002-5494-685X"},"institutions":[{"id":"https://openalex.org/I204722609","display_name":"Queen's University","ror":"https://ror.org/02y72wh86","country_code":"CA","type":"education","lineage":["https://openalex.org/I204722609"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Filipe Roseiro C\u00f4go","raw_affiliation_strings":["Software Analysis and Intelligence Laboratory (SAIL), School of Computing, Queen&#x2019;s University, Kingston, ON, Canada","Software Analysis and Intelligence Lab (SAIL), School of Computing, Queen&#x2019;s University, Kingston, ON, Canada"],"raw_orcid":"https://orcid.org/0000-0002-5494-685X","affiliations":[{"raw_affiliation_string":"Software Analysis and Intelligence Laboratory (SAIL), School of Computing, Queen&#x2019;s University, Kingston, ON, Canada","institution_ids":["https://openalex.org/I204722609"]},{"raw_affiliation_string":"Software Analysis and Intelligence Lab (SAIL), School of Computing, Queen&#x2019;s University, Kingston, ON, Canada","institution_ids":["https://openalex.org/I204722609"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058164381","display_name":"Bram Adams","orcid":"https://orcid.org/0000-0001-7213-4006"},"institutions":[{"id":"https://openalex.org/I204722609","display_name":"Queen's University","ror":"https://ror.org/02y72wh86","country_code":"CA","type":"education","lineage":["https://openalex.org/I204722609"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Bram Adams","raw_affiliation_strings":["Software Analysis and Intelligence Laboratory (SAIL), School of Computing, Queen&#x2019;s University, Kingston, ON, Canada","Software Analysis and Intelligence Lab (SAIL), School of Computing, Queen&#x2019;s University, Kingston, ON, Canada"],"raw_orcid":"https://orcid.org/0000-0001-7213-4006","affiliations":[{"raw_affiliation_string":"Software Analysis and Intelligence Laboratory (SAIL), School of Computing, Queen&#x2019;s University, Kingston, ON, Canada","institution_ids":["https://openalex.org/I204722609"]},{"raw_affiliation_string":"Software Analysis and Intelligence Lab (SAIL), School of Computing, Queen&#x2019;s University, Kingston, ON, Canada","institution_ids":["https://openalex.org/I204722609"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5091586373","display_name":"Ahmed E. Hassan","orcid":"https://orcid.org/0000-0001-7749-5513"},"institutions":[{"id":"https://openalex.org/I204722609","display_name":"Queen's University","ror":"https://ror.org/02y72wh86","country_code":"CA","type":"education","lineage":["https://openalex.org/I204722609"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Ahmed E. Hassan","raw_affiliation_strings":["Software Analysis and Intelligence Laboratory (SAIL), School of Computing, Queen&#x2019;s University, Kingston, ON, Canada","Software Analysis and Intelligence Lab (SAIL), School of Computing, Queen&#x2019;s University, Kingston, ON, Canada"],"raw_orcid":"https://orcid.org/0000-0001-7749-5513","affiliations":[{"raw_affiliation_string":"Software Analysis and Intelligence Laboratory (SAIL), School of Computing, Queen&#x2019;s University, Kingston, ON, Canada","institution_ids":["https://openalex.org/I204722609"]},{"raw_affiliation_string":"Software Analysis and Intelligence Lab (SAIL), School of Computing, Queen&#x2019;s University, Kingston, ON, Canada","institution_ids":["https://openalex.org/I204722609"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I204722609"],"apc_list":null,"apc_paid":null,"fwci":2.5213,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.88851212,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":97},"biblio":{"volume":"51","issue":"4","first_page":"929","last_page":"946"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11994","display_name":"Human Resource Development and Performance Evaluation","score":0.07980000227689743,"subfield":{"id":"https://openalex.org/subfields/3202","display_name":"Applied Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11994","display_name":"Human Resource Development and Performance Evaluation","score":0.07980000227689743,"subfield":{"id":"https://openalex.org/subfields/3202","display_name":"Applied Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13163","display_name":"Organizational Learning and Leadership","score":0.0722000002861023,"subfield":{"id":"https://openalex.org/subfields/1407","display_name":"Organizational Behavior and Human Resource Management"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10395","display_name":"Construction Project Management and Performance","score":0.06480000168085098,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8050539493560791},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.6475394368171692},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.6282855868339539},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.6145636439323425},{"id":"https://openalex.org/keywords/foundation","display_name":"Foundation (evidence)","score":0.5117034912109375},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.3653470277786255},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.22069919109344482}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8050539493560791},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.6475394368171692},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.6282855868339539},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.6145636439323425},{"id":"https://openalex.org/C2780966255","wikidata":"https://www.wikidata.org/wiki/Q5474306","display_name":"Foundation (evidence)","level":2,"score":0.5117034912109375},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.3653470277786255},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.22069919109344482},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tse.2025.3533972","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2025.3533972","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":80,"referenced_works":["https://openalex.org/W1981425990","https://openalex.org/W2019955851","https://openalex.org/W2045336717","https://openalex.org/W2051275384","https://openalex.org/W2063948020","https://openalex.org/W2107006322","https://openalex.org/W2137296056","https://openalex.org/W2147810104","https://openalex.org/W2160517961","https://openalex.org/W2164577291","https://openalex.org/W2763110165","https://openalex.org/W2780783514","https://openalex.org/W2890432166","https://openalex.org/W2898017895","https://openalex.org/W2900374634","https://openalex.org/W2953723845","https://openalex.org/W2955656327","https://openalex.org/W2970819455","https://openalex.org/W3016821494","https://openalex.org/W3019789734","https://openalex.org/W3034368386","https://openalex.org/W3090913539","https://openalex.org/W3091625722","https://openalex.org/W3103940211","https://openalex.org/W3137147200","https://openalex.org/W3154151289","https://openalex.org/W3195548736","https://openalex.org/W3196126762","https://openalex.org/W3202595962","https://openalex.org/W3209295695","https://openalex.org/W3211022409","https://openalex.org/W4246121531","https://openalex.org/W4286532195","https://openalex.org/W4288064569","https://openalex.org/W4289828962","https://openalex.org/W4308885736","https://openalex.org/W4309764786","https://openalex.org/W4313016430","https://openalex.org/W4319337589","https://openalex.org/W4324102217","https://openalex.org/W4360995249","https://openalex.org/W4362659486","https://openalex.org/W4378189609","https://openalex.org/W4380996167","https://openalex.org/W4384261711","https://openalex.org/W4384345745","https://openalex.org/W4394745423","https://openalex.org/W4396832834","https://openalex.org/W4400491795","https://openalex.org/W4401878108","https://openalex.org/W4402684050","https://openalex.org/W4404181169","https://openalex.org/W4404518481","https://openalex.org/W4404781650","https://openalex.org/W4404782950","https://openalex.org/W6729254871","https://openalex.org/W6756223733","https://openalex.org/W6764357534","https://openalex.org/W6764990469","https://openalex.org/W6777615688","https://openalex.org/W6780779376","https://openalex.org/W6798182279","https://openalex.org/W6800751262","https://openalex.org/W6840081018","https://openalex.org/W6845441306","https://openalex.org/W6851592950","https://openalex.org/W6852874933","https://openalex.org/W6853906931","https://openalex.org/W6857856913","https://openalex.org/W6857916434","https://openalex.org/W6859010671","https://openalex.org/W6861607970","https://openalex.org/W6862331864","https://openalex.org/W6862631004","https://openalex.org/W6864366853","https://openalex.org/W6867022496","https://openalex.org/W6873297891","https://openalex.org/W6873398335","https://openalex.org/W6873784770","https://openalex.org/W7019082250"],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W2381393187","https://openalex.org/W2332779545","https://openalex.org/W1981780420","https://openalex.org/W2182707996","https://openalex.org/W45233828","https://openalex.org/W2964988449","https://openalex.org/W2397952901","https://openalex.org/W2029380707","https://openalex.org/W188202134"],"abstract_inverted_index":{"Foundation":[0],"models":[1,7],"(FM),":[2],"such":[3,27],"as":[4,28],"large":[5],"language":[6],"(LLMs),":[8],"which":[9],"are":[10],"large-scale":[11],"machine":[12],"learning":[13],"(ML)":[14],"models,":[15],"have":[16,41],"demonstrated":[17],"remarkable":[18],"adaptability":[19],"in":[20,106,153,200,214],"various":[21],"downstream":[22],"software":[23,34],"engineering":[24],"(SE)":[25],"tasks,":[26],"code":[29,31],"completion,":[30],"understanding,":[32],"and":[33,50,60,71,79,111,115,144,151,166,175,185,212,222,228],"development.":[35],"As":[36,88],"a":[37,89,177,219],"result,":[38],"FM":[39,69,77,86,103,130,226],"leaderboards":[40,78,104,131],"become":[42],"essential":[43],"tools":[44],"for":[45,56,68,117,225],"SE":[46,206],"teams":[47,207],"to":[48,83,127,147,159],"compare":[49],"select":[51],"the":[52,63,74,182],"best":[53],"third-party":[54],"FMs":[55],"their":[57,149,161,186],"specific":[58],"products":[59],"purposes.":[61],"However,":[62],"lack":[64],"of":[65,76,197],"standardized":[66],"guidelines":[67],"evaluation":[70],"comparison":[72,227],"threatens":[73],"transparency":[75],"limits":[80],"stakeholders\u2019":[81],"ability":[82],"perform":[84],"effective":[85],"selection.":[87,229],"first":[90],"step":[91],"towards":[92],"addressing":[93],"this":[94,122],"challenge,":[95],"our":[96],"research":[97],"focuses":[98],"on":[99],"understanding":[100],"how":[101],"these":[102,189,204],"operate":[105],"real-world":[107],"scenarios":[108],"(\u201cleaderboard":[109,119],"operations\u201d)":[110],"identifying":[112],"potential":[113],"pitfalls":[114],"areas":[116],"improvement":[118],"smells\u201d).":[120],"In":[121],"regard,":[123],"we":[124,169],"collect":[125],"up":[126],"<inline-formula><tex-math":[128],"notation=\"LaTeX\">$1,045$</tex-math></inline-formula>":[129],"from":[132],"five":[133,171],"different":[134],"sources:":[135],"GitHub,":[136],"Hugging":[137],"Face":[138],"Spaces,":[139],"Papers":[140],"With":[141],"Code,":[142],"spreadsheet":[143],"independent":[145],"platform,":[146],"examine":[148],"documentation":[150],"engage":[152],"direct":[154],"communication":[155],"with":[156],"leaderboard":[157,198],"operators":[158],"understand":[160],"workflows.":[162,190],"Through":[163],"card":[164],"sorting":[165],"negotiated":[167],"agreement,":[168],"identify":[170,193],"distinct":[172],"workflow":[173],"patterns":[174],"develop":[176],"domain":[178],"model":[179],"that":[180],"captures":[181],"key":[183],"components":[184],"interactions":[187],"within":[188],"We":[191],"then":[192],"eight":[194],"unique":[195],"types":[196],"smells":[199],"LBOps.":[201],"By":[202],"mitigating":[203],"smells,":[205],"can":[208],"improve":[209],"transparency,":[210],"accountability,":[211],"collaboration":[213],"current":[215],"LBOps":[216],"practices,":[217],"fostering":[218],"more":[220],"robust":[221],"responsible":[223],"ecosystem":[224]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}