{"id":"https://openalex.org/W4303648020","doi":"https://doi.org/10.48550/arxiv.2210.01970","title":"Evaluate & Evaluation on the Hub: Better Best Practices for Data and Model Measurements","display_name":"Evaluate & Evaluation on the Hub: Better Best Practices for Data and Model Measurements","publication_year":2022,"publication_date":"2022-09-30","ids":{"openalex":"https://openalex.org/W4303648020","doi":"https://doi.org/10.48550/arxiv.2210.01970"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2210.01970","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2210.01970","pdf_url":"https://arxiv.org/pdf/2210.01970","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2210.01970","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5008355834","display_name":"Leandro von Werra","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"von Werra, Leandro","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019163452","display_name":"Lewis Tunstall","orcid":"https://orcid.org/0000-0003-3309-5520"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tunstall, Lewis","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022414335","display_name":"Abhishek Thakur","orcid":"https://orcid.org/0000-0002-9955-9693"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Thakur, Abhishek","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091714241","display_name":"Alexandra Sasha Luccioni","orcid":"https://orcid.org/0000-0001-6238-7050"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luccioni, Alexandra Sasha","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026811404","display_name":"Tristan Thrush","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Thrush, Tristan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055847471","display_name":"Aleksandra Piktus","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Piktus, Aleksandra","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078498271","display_name":"F\u00e9lix Marty","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Marty, Felix","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052900932","display_name":"Nazneen Fatema Rajani","orcid":"https://orcid.org/0000-0001-6301-1960"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rajani, Nazneen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038928331","display_name":"Victor Mustar","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mustar, Victor","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044621620","display_name":"Helen L. Ngo","orcid":"https://orcid.org/0000-0002-6292-1695"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ngo, Helen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089085100","display_name":"Omar Sanseviero","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sanseviero, Omar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075136074","display_name":"Mario \u0160a\u0161ko","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"\u0160a\u0161ko, Mario","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060240092","display_name":"Albert Villanova","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Villanova, Albert","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074885928","display_name":"Quentin Lhoest","orcid":"https://orcid.org/0009-0000-7660-3420"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lhoest, Quentin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038505596","display_name":"Julien Chaumond","orcid":"https://orcid.org/0000-0003-3188-1616"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chaumond, Julien","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046235098","display_name":"Margaret Mitchell","orcid":"https://orcid.org/0000-0001-7043-6545"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mitchell, Margaret","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062546146","display_name":"Alexander M. Rush","orcid":"https://orcid.org/0000-0002-9900-1606"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rush, Alexander M.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078865608","display_name":"Thomas Wolf","orcid":"https://orcid.org/0000-0002-7134-7314"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wolf, Thomas","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5016956470","display_name":"Douwe Kiela","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kiela, Douwe","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":5,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.992900013923645,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.992900013923645,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9922999739646912,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.9732000231742859,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/implementation","display_name":"Implementation","score":0.8416047096252441},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8077198266983032},{"id":"https://openalex.org/keywords/documentation","display_name":"Documentation","score":0.7016593813896179},{"id":"https://openalex.org/keywords/variety","display_name":"Variety (cybernetics)","score":0.6217440962791443},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.6053154468536377},{"id":"https://openalex.org/keywords/best-practice","display_name":"Best practice","score":0.5649017095565796},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5607286095619202},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5241230130195618},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.5085211992263794},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.4857187867164612},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3866727650165558},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3619147539138794},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.34885352849960327},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.27143311500549316},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.1264464259147644}],"concepts":[{"id":"https://openalex.org/C26713055","wikidata":"https://www.wikidata.org/wiki/Q245962","display_name":"Implementation","level":2,"score":0.8416047096252441},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8077198266983032},{"id":"https://openalex.org/C56666940","wikidata":"https://www.wikidata.org/wiki/Q788790","display_name":"Documentation","level":2,"score":0.7016593813896179},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.6217440962791443},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.6053154468536377},{"id":"https://openalex.org/C184356942","wikidata":"https://www.wikidata.org/wiki/Q830382","display_name":"Best practice","level":2,"score":0.5649017095565796},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5607286095619202},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5241230130195618},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.5085211992263794},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4857187867164612},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3866727650165558},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3619147539138794},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.34885352849960327},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.27143311500549316},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.1264464259147644},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2210.01970","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2210.01970","pdf_url":"https://arxiv.org/pdf/2210.01970","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2210.01970","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2210.01970","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2210.01970","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2210.01970","pdf_url":"https://arxiv.org/pdf/2210.01970","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2002383399","https://openalex.org/W2618286804","https://openalex.org/W2329643025","https://openalex.org/W644644594","https://openalex.org/W2002770077","https://openalex.org/W2506073049","https://openalex.org/W3131163342","https://openalex.org/W642180557","https://openalex.org/W2092256833","https://openalex.org/W2052780847"],"abstract_inverted_index":{"Evaluation":[0,29,123,154],"is":[1,11,48,66,115,158],"a":[2,12,49,96,127,152],"key":[3],"part":[4],"of":[5,14,35,41,60,70,85,98,134,151],"machine":[6],"learning":[7],"(ML),":[8],"yet":[9],"there":[10],"lack":[13],"support":[15,52,68],"and":[16,22,28,43,58,62,73,78,100,104,111,138],"tooling":[17],"to":[18,37,51,67,81,107],"enable":[19],"its":[20],"informed":[21],"systematic":[23],"practice.":[24],"We":[25],"introduce":[26,122],"Evaluate":[27,47],"on":[30,124,141,155],"the":[31,39,75,105,125,131,142,149,156],"Hub":[32,157],"--a":[33],"set":[34],"tools":[36],"facilitate":[38],"evaluation":[40,76,80,133],"models":[42,137],"datasets":[44,140],"in":[45],"ML.":[46],"library":[50,114],"best":[53],"practices":[54],"for":[55,95,146],"measurements,":[56],"metrics,":[57],"comparisons":[59],"data":[61],"models.":[63],"Its":[64],"goal":[65],"reproducibility":[69],"evaluation,":[71],"centralize":[72],"document":[74],"process,":[77],"broaden":[79],"cover":[82],"more":[83],"facets":[84],"model":[86],"performance.":[87],"It":[88],"includes":[89],"over":[90,135],"50":[91],"efficient":[92],"canonical":[93],"implementations":[94,110],"variety":[97],"domains":[99],"scenarios,":[101],"interactive":[102],"documentation,":[103],"ability":[106],"easily":[108],"share":[109],"outcomes.":[112],"The":[113],"available":[116,159],"at":[117,148,160],"https://github.com/huggingface/evaluate.":[118],"In":[119],"addition,":[120],"we":[121],"Hub,":[126,145],"platform":[128],"that":[129],"enables":[130],"large-scale":[132],"75,000":[136],"11,000":[139],"Hugging":[143],"Face":[144],"free,":[147],"click":[150],"button.":[153],"https://huggingface.co/autoevaluate.":[161]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":4}],"updated_date":"2026-08-11T07:18:39.950985","created_date":"2022-10-08T00:00:00"}