{"id":"https://openalex.org/W4393241167","doi":"https://doi.org/10.1109/tkde.2024.3382002","title":"A Survey for Federated Learning Evaluations: Goals and Measures","display_name":"A Survey for Federated Learning Evaluations: Goals and Measures","publication_year":2024,"publication_date":"2024-03-27","ids":{"openalex":"https://openalex.org/W4393241167","doi":"https://doi.org/10.1109/tkde.2024.3382002"},"language":"en","primary_location":{"id":"doi:10.1109/tkde.2024.3382002","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tkde.2024.3382002","pdf_url":null,"source":{"id":"https://openalex.org/S30698027","display_name":"IEEE Transactions on Knowledge and Data Engineering","issn_l":"1041-4347","issn":["1041-4347","1558-2191","2326-3865"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Knowledge and Data Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5057997406","display_name":"Di Chai","orcid":"https://orcid.org/0000-0003-4810-0644"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":true,"raw_author_name":"Di Chai","raw_affiliation_strings":["Hong Kong University of Science and Technology, Hong Kong, China","Hong Kong University of Science and Technology, Hong Kong SAR, China"],"affiliations":[{"raw_affiliation_string":"Hong Kong University of Science and Technology, Hong Kong, China","institution_ids":["https://openalex.org/I200769079"]},{"raw_affiliation_string":"Hong Kong University of Science and Technology, Hong Kong SAR, China","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055087680","display_name":"Leye Wang","orcid":"https://orcid.org/0000-0002-7627-8485"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I4210128818","display_name":"Institute of Software","ror":"https://ror.org/033dfsn42","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210128818"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Leye Wang","raw_affiliation_strings":["Key Lab of High Confidence Software Technologies, Ministry of Education, Beijing, China","School of Computer Science, Peking University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Key Lab of High Confidence Software Technologies, Ministry of Education, Beijing, China","institution_ids":["https://openalex.org/I4210128818"]},{"raw_affiliation_string":"School of Computer Science, Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100693653","display_name":"Yang Liu","orcid":"https://orcid.org/0000-0002-4393-1791"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Liu Yang","raw_affiliation_strings":["Hong Kong University of Science and Technology, Hong Kong, China","Hong Kong University of Science and Technology, Hong Kong SAR, China"],"affiliations":[{"raw_affiliation_string":"Hong Kong University of Science and Technology, Hong Kong, China","institution_ids":["https://openalex.org/I200769079"]},{"raw_affiliation_string":"Hong Kong University of Science and Technology, Hong Kong SAR, China","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071584772","display_name":"Junxue Zhang","orcid":"https://orcid.org/0000-0001-6926-7801"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Junxue Zhang","raw_affiliation_strings":["Hong Kong University of Science and Technology, Hong Kong, China","Hong Kong University of Science and Technology, Hong Kong SAR, China"],"affiliations":[{"raw_affiliation_string":"Hong Kong University of Science and Technology, Hong Kong, China","institution_ids":["https://openalex.org/I200769079"]},{"raw_affiliation_string":"Hong Kong University of Science and Technology, Hong Kong SAR, China","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100438001","display_name":"Kai Chen","orcid":"https://orcid.org/0000-0003-2587-6028"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Kai Chen","raw_affiliation_strings":["Hong Kong University of Science and Technology, Hong Kong, China","Hong Kong University of Science and Technology, Hong Kong SAR, China"],"affiliations":[{"raw_affiliation_string":"Hong Kong University of Science and Technology, Hong Kong, China","institution_ids":["https://openalex.org/I200769079"]},{"raw_affiliation_string":"Hong Kong University of Science and Technology, Hong Kong SAR, China","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100636286","display_name":"Qiang Yang","orcid":"https://orcid.org/0000-0001-5059-8360"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Qiang Yang","raw_affiliation_strings":["Hong Kong University of Science and Technology, Hong Kong, China","Webank, Shenzhen, China","Hong Kong University of Science and Technology, Hong Kong SAR, China"],"affiliations":[{"raw_affiliation_string":"Hong Kong University of Science and Technology, Hong Kong, China","institution_ids":["https://openalex.org/I200769079"]},{"raw_affiliation_string":"Webank, Shenzhen, China","institution_ids":[]},{"raw_affiliation_string":"Hong Kong University of Science and Technology, Hong Kong SAR, China","institution_ids":["https://openalex.org/I200769079"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5057997406"],"corresponding_institution_ids":["https://openalex.org/I200769079"],"apc_list":null,"apc_paid":null,"fwci":14.1992,"has_fulltext":false,"cited_by_count":43,"citation_normalized_percentile":{"value":0.99130213,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":"36","issue":"10","first_page":"5007","last_page":"5024"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.991100013256073,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11045","display_name":"Privacy, Security, and Data Protection","score":0.9904000163078308,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8478212356567383},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.47142666578292847},{"id":"https://openalex.org/keywords/open-research","display_name":"Open research","score":0.45671147108078003},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3340461254119873},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.32871609926223755},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.22715795040130615}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8478212356567383},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.47142666578292847},{"id":"https://openalex.org/C2778464652","wikidata":"https://www.wikidata.org/wiki/Q309849","display_name":"Open research","level":2,"score":0.45671147108078003},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3340461254119873},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.32871609926223755},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.22715795040130615}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tkde.2024.3382002","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tkde.2024.3382002","pdf_url":null,"source":{"id":"https://openalex.org/S30698027","display_name":"IEEE Transactions on Knowledge and Data Engineering","issn_l":"1041-4347","issn":["1041-4347","1558-2191","2326-3865"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Knowledge and Data Engineering","raw_type":"journal-article"},{"id":"pmh:oai:repository.hkust.edu.hk:1783.1-138671","is_oa":false,"landing_page_url":"http://repository.hkust.edu.hk/ir/Record/1783.1-138671","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":132,"referenced_works":["https://openalex.org/W1574680346","https://openalex.org/W1969009977","https://openalex.org/W2013317087","https://openalex.org/W2051267297","https://openalex.org/W2336650964","https://openalex.org/W2406176316","https://openalex.org/W2487770199","https://openalex.org/W2535690855","https://openalex.org/W2701059868","https://openalex.org/W2767079719","https://openalex.org/W2773194476","https://openalex.org/W2807006176","https://openalex.org/W2899461894","https://openalex.org/W2903471046","https://openalex.org/W2912213068","https://openalex.org/W2930926105","https://openalex.org/W2955213239","https://openalex.org/W2989289980","https://openalex.org/W3014718068","https://openalex.org/W3015335668","https://openalex.org/W3015636663","https://openalex.org/W3016378036","https://openalex.org/W3016632787","https://openalex.org/W3017915781","https://openalex.org/W3044211235","https://openalex.org/W3046653923","https://openalex.org/W3047789363","https://openalex.org/W3088731039","https://openalex.org/W3091870957","https://openalex.org/W3097060730","https://openalex.org/W3100035104","https://openalex.org/W3103245149","https://openalex.org/W3103401990","https://openalex.org/W3106094943","https://openalex.org/W3109000088","https://openalex.org/W3109176830","https://openalex.org/W3123459983","https://openalex.org/W3135231128","https://openalex.org/W3135472452","https://openalex.org/W3138090943","https://openalex.org/W3138597937","https://openalex.org/W3155912831","https://openalex.org/W3159080474","https://openalex.org/W3169231731","https://openalex.org/W3175418606","https://openalex.org/W3177095755","https://openalex.org/W3180608480","https://openalex.org/W3182125009","https://openalex.org/W3192072825","https://openalex.org/W3196371845","https://openalex.org/W3196965143","https://openalex.org/W3198746473","https://openalex.org/W3201788422","https://openalex.org/W3206887799","https://openalex.org/W3209696639","https://openalex.org/W3211660277","https://openalex.org/W3212079419","https://openalex.org/W3216652988","https://openalex.org/W4205487313","https://openalex.org/W4210467203","https://openalex.org/W4214758645","https://openalex.org/W4221156096","https://openalex.org/W4225821859","https://openalex.org/W4226307069","https://openalex.org/W4226461837","https://openalex.org/W4229029907","https://openalex.org/W4281848858","https://openalex.org/W4283320273","https://openalex.org/W4285337687","https://openalex.org/W4285601288","https://openalex.org/W4287179788","https://openalex.org/W4288057790","https://openalex.org/W4288057793","https://openalex.org/W4289446312","https://openalex.org/W4297060071","https://openalex.org/W4308632285","https://openalex.org/W4308643055","https://openalex.org/W4308643126","https://openalex.org/W4310822417","https://openalex.org/W4311080353","https://openalex.org/W4312395750","https://openalex.org/W4312646383","https://openalex.org/W4362515174","https://openalex.org/W4372348664","https://openalex.org/W4380433128","https://openalex.org/W4381329494","https://openalex.org/W6703420464","https://openalex.org/W6713134421","https://openalex.org/W6728757088","https://openalex.org/W6738383168","https://openalex.org/W6746388805","https://openalex.org/W6752029299","https://openalex.org/W6752600739","https://openalex.org/W6757139170","https://openalex.org/W6759238902","https://openalex.org/W6762879059","https://openalex.org/W6764838729","https://openalex.org/W6765541894","https://openalex.org/W6770634426","https://openalex.org/W6772307254","https://openalex.org/W6773976177","https://openalex.org/W6775563089","https://openalex.org/W6779174293","https://openalex.org/W6780897730","https://openalex.org/W6781175393","https://openalex.org/W6781232731","https://openalex.org/W6783533280","https://openalex.org/W6784311791","https://openalex.org/W6784336702","https://openalex.org/W6785264981","https://openalex.org/W6786060194","https://openalex.org/W6789100154","https://openalex.org/W6789305514","https://openalex.org/W6790212648","https://openalex.org/W6790438242","https://openalex.org/W6795916790","https://openalex.org/W6797586921","https://openalex.org/W6802646539","https://openalex.org/W6803730191","https://openalex.org/W6803975504","https://openalex.org/W6805407591","https://openalex.org/W6810249531","https://openalex.org/W6838423293","https://openalex.org/W6838967081","https://openalex.org/W6839716622","https://openalex.org/W6839799260","https://openalex.org/W6840391442","https://openalex.org/W6847350444","https://openalex.org/W6850227834","https://openalex.org/W6852464802","https://openalex.org/W6991308149","https://openalex.org/W7056673059"],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W3046775127","https://openalex.org/W3107602296","https://openalex.org/W4394896187","https://openalex.org/W3170094116","https://openalex.org/W4386462264","https://openalex.org/W4364306694","https://openalex.org/W4312192474","https://openalex.org/W4283697347"],"abstract_inverted_index":{"Evaluation":[0],"is":[1,18,41],"a":[2,9,19,94],"systematic":[3],"approach":[4],"to":[5,30,44],"assessing":[6],"how":[7],"well":[8],"system":[10],"achieves":[11],"its":[12,45],"intended":[13],"purpose.":[14],"Federated":[15],"learning":[16,25],"(FL)":[17],"novel":[20],"paradigm":[21],"for":[22,79,100,120],"privacy-preserving":[23],"machine":[24],"that":[26,92],"allows":[27],"multiple":[28],"parties":[29],"collaboratively":[31],"train":[32],"models":[33],"without":[34],"sharing":[35],"sensitive":[36],"data.":[37],"However,":[38],"evaluating":[39],"FL":[40,101,121],"challenging":[42],"due":[43],"interdisciplinary":[46],"nature":[47],"and":[48,55,72,96,109,116],"diverse":[49],"goals,":[50],"such":[51],"as":[52],"utility,":[53,107],"efficiency,":[54,108],"security.":[56,110],"In":[57],"this":[58],"survey,":[59],"we":[60,112],"first":[61],"review":[62],"the":[63,69,75],"major":[64],"evaluation":[65,76,98],"goals":[66],"adopted":[67],"in":[68,103],"existing":[70],"studies":[71],"then":[73],"explore":[74],"metrics":[77],"used":[78],"each":[80],"goal.":[81],"We":[82],"also":[83],"introduce":[84],"<italic":[85],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[86],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">FedEval</i>":[87],",":[88],"an":[89],"open-source":[90],"platform":[91],"provides":[93],"standardized":[95],"comprehensive":[97],"framework":[99],"algorithms":[102],"terms":[104],"of":[105],"their":[106],"Finally,":[111],"discuss":[113],"several":[114],"challenges":[115],"future":[117],"research":[118],"directions":[119],"evaluation.":[122]},"counts_by_year":[{"year":2026,"cited_by_count":8},{"year":2025,"cited_by_count":27},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":1}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2025-10-10T00:00:00"}