{"id":"https://openalex.org/W3192401439","doi":"https://doi.org/10.48550/arxiv.2003.12206","title":"Improving Reproducibility in Machine Learning Research (A Report from the NeurIPS 2019 Reproducibility Program)","display_name":"Improving Reproducibility in Machine Learning Research (A Report from the NeurIPS 2019 Reproducibility Program)","publication_year":2020,"publication_date":"2020-03-27","ids":{"openalex":"https://openalex.org/W3192401439","doi":"https://doi.org/10.48550/arxiv.2003.12206","mag":"3192401439"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2003.12206","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2003.12206","pdf_url":"https://arxiv.org/pdf/2003.12206","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2003.12206","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5080591144","display_name":"Jo\u00eblle Pineau","orcid":"https://orcid.org/0000-0003-0747-7250","observed_orcids":["https://orcid.org/0000-0003-0747-7250"]},"institutions":[{"id":"https://openalex.org/I109736498","display_name":"Canadian Institute for Advanced Research","ror":"https://ror.org/01sdtdd95","country_code":"CA","type":"facility","lineage":["https://openalex.org/I109736498"]},{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]},{"id":"https://openalex.org/I4210164802","display_name":"Mila - Quebec Artificial Intelligence Institute","ror":"https://ror.org/05c22rx21","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210164802"]},{"id":"https://openalex.org/I5023651","display_name":"McGill University","ror":"https://ror.org/01pxwe438","country_code":"CA","type":"education","lineage":["https://openalex.org/I5023651"]}],"countries":["CA","US"],"is_corresponding":false,"raw_author_name":"Pineau, Joelle","raw_affiliation_strings":["School of Computer Science , McGill University (Mila) Facebook AI Research CIFAR","Facebook AI Research [Montr\u00e9al]","McGill University = Universit\u00e9 McGill [Montr\u00e9al, Canada]"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science , McGill University (Mila) Facebook AI Research CIFAR","institution_ids":["https://openalex.org/I109736498","https://openalex.org/I4210114444","https://openalex.org/I4210164802","https://openalex.org/I5023651"]},{"raw_affiliation_string":"Facebook AI Research [Montr\u00e9al]","institution_ids":["https://openalex.org/I4210114444"]},{"raw_affiliation_string":"McGill University = Universit\u00e9 McGill [Montr\u00e9al, Canada]","institution_ids":["https://openalex.org/I5023651"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002591360","display_name":"Philippe Vincent\u2010Lamarre","orcid":"https://orcid.org/0000-0002-1358-8937","observed_orcids":["https://orcid.org/0000-0002-1358-8937"]},"institutions":[{"id":"https://openalex.org/I109736498","display_name":"Canadian Institute for Advanced Research","ror":"https://ror.org/01sdtdd95","country_code":"CA","type":"facility","lineage":["https://openalex.org/I109736498"]},{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]},{"id":"https://openalex.org/I4210164802","display_name":"Mila - Quebec Artificial Intelligence Institute","ror":"https://ror.org/05c22rx21","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210164802"]},{"id":"https://openalex.org/I5023651","display_name":"McGill University","ror":"https://ror.org/01pxwe438","country_code":"CA","type":"education","lineage":["https://openalex.org/I5023651"]},{"id":"https://openalex.org/I70931966","display_name":"Universit\u00e9 de Montr\u00e9al","ror":"https://ror.org/0161xgx34","country_code":"CA","type":"education","lineage":["https://openalex.org/I70931966"]}],"countries":["CA","US"],"is_corresponding":false,"raw_author_name":"Vincent-Lamarre, Philippe","raw_affiliation_strings":["Ecole de biblioth\u00e8conomie et des sciences de l'information , Universit\u00e9 de Montr\u00e9al","School of Computer Science , McGill University (Mila) Facebook AI Research CIFAR","\u00c9cole de biblioth\u00e9conomie et des sciences de l\u2019information"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ecole de biblioth\u00e8conomie et des sciences de l'information , Universit\u00e9 de Montr\u00e9al","institution_ids":["https://openalex.org/I70931966"]},{"raw_affiliation_string":"School of Computer Science , McGill University (Mila) Facebook AI Research CIFAR","institution_ids":["https://openalex.org/I109736498","https://openalex.org/I4210114444","https://openalex.org/I4210164802","https://openalex.org/I5023651"]},{"raw_affiliation_string":"\u00c9cole de biblioth\u00e9conomie et des sciences de l\u2019information","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019642748","display_name":"K. Sinha","orcid":"https://orcid.org/0009-0009-2325-6846","observed_orcids":["https://orcid.org/0009-0009-2325-6846"]},"institutions":[{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]},{"id":"https://openalex.org/I5023651","display_name":"McGill University","ror":"https://ror.org/01pxwe438","country_code":"CA","type":"education","lineage":["https://openalex.org/I5023651"]}],"countries":["CA","US"],"is_corresponding":false,"raw_author_name":"Sinha, Koustuv","raw_affiliation_strings":["School of Computer Science , McGill University (Mila) Facebook AI Research","Facebook AI Research [Montr\u00e9al]","McGill University = Universit\u00e9 McGill [Montr\u00e9al, Canada]"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science , McGill University (Mila) Facebook AI Research","institution_ids":["https://openalex.org/I5023651"]},{"raw_affiliation_string":"Facebook AI Research [Montr\u00e9al]","institution_ids":["https://openalex.org/I4210114444"]},{"raw_affiliation_string":"McGill University = Universit\u00e9 McGill [Montr\u00e9al, Canada]","institution_ids":["https://openalex.org/I5023651"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068542997","display_name":"Vincent Larivi\u00e8re","orcid":"https://orcid.org/0000-0002-2733-0689","observed_orcids":["https://orcid.org/0000-0002-2733-0689"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Larivi\u00e8re, Vincent","raw_affiliation_strings":["\u00c9cole de biblioth\u00e9conomie et des sciences de l\u2019information"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u00c9cole de biblioth\u00e9conomie et des sciences de l\u2019information","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078135763","display_name":"Alina Beygelzimer","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210134091","display_name":"Yahoo (United States)","ror":"https://ror.org/040dkzz12","country_code":"US","type":"company","lineage":["https://openalex.org/I4210134091"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Beygelzimer, Alina","raw_affiliation_strings":["Yahoo! Research Florence d'Alch\u00e9-Buc","Yahoo Labs., New York#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yahoo! Research Florence d'Alch\u00e9-Buc","institution_ids":[]},{"raw_affiliation_string":"Yahoo Labs., New York#TAB#","institution_ids":["https://openalex.org/I4210134091"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066766964","display_name":"Florence d\u2019Alch\u00e9\u2013Buc","orcid":"https://orcid.org/0000-0002-8353-0589","observed_orcids":["https://orcid.org/0000-0002-8353-0589"]},"institutions":[{"id":"https://openalex.org/I4210145102","display_name":"Institut Polytechnique de Paris","ror":"https://ror.org/042tfbd02","country_code":"FR","type":"education","lineage":["https://openalex.org/I4210145102"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"d'Alch\u00e9-Buc, Florence","raw_affiliation_strings":["D\u00e9partement Images, Donn\u00e9es, Signal","Institut Polytechnique de Paris","Signal, Statistique et Apprentissage"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"D\u00e9partement Images, Donn\u00e9es, Signal","institution_ids":[]},{"raw_affiliation_string":"Institut Polytechnique de Paris","institution_ids":["https://openalex.org/I4210145102"]},{"raw_affiliation_string":"Signal, Statistique et Apprentissage","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068568859","display_name":"Emily B. Fox","orcid":"https://orcid.org/0000-0003-3188-9685","observed_orcids":["https://orcid.org/0000-0003-3188-9685"]},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]},{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Fox, Emily","raw_affiliation_strings":["Apple Inc","Paul G. Allen School of Computer Science and Engineering [Seattle]"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Apple Inc","institution_ids":["https://openalex.org/I4210153776"]},{"raw_affiliation_string":"Paul G. Allen School of Computer Science and Engineering [Seattle]","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5019862529","display_name":"Hugo Larochelle","orcid":"https://orcid.org/0000-0002-2960-9288","observed_orcids":["https://orcid.org/0000-0002-2960-9288"]},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]},{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Larochelle, Hugo","raw_affiliation_strings":["University of Washington Apple","Brain team [Paris]"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Washington Apple","institution_ids":["https://openalex.org/I201448701","https://openalex.org/I4210153776"]},{"raw_affiliation_string":"Brain team [Paris]","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":9,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":317,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":"22","issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9150000214576721,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9150000214576721,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13607","display_name":"Academic Publishing and Open Access","score":0.02879999950528145,"subfield":{"id":"https://openalex.org/subfields/3309","display_name":"Library and Information Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11937","display_name":"Research Data Management Practices","score":0.011800000444054604,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reproducibility","display_name":"reproducibility","score":0.9350000023841858},{"id":"https://openalex.org/keywords/machine-learning-research","display_name":"machine learning research","score":0.9549999833106995},{"id":"https://openalex.org/keywords/reproducibility-challenge","display_name":"reproducibility challenge","score":0.9879999756813049},{"id":"https://openalex.org/keywords/research-reliability","display_name":"research reliability","score":0.7059999704360962},{"id":"https://openalex.org/keywords/open-research","display_name":"open research","score":0.6859999895095825},{"id":"https://openalex.org/keywords/experimental-workflow","display_name":"experimental workflow","score":0.6200000047683716}],"concepts":[{"id":"https://openalex.org/C9893847","wikidata":"https://www.wikidata.org/wiki/Q1425625","display_name":"Reproducibility","level":2,"score":0.9703805446624756},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.422443687915802},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.12595131993293762},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.10493797063827515}],"mesh":[],"locations_count":5,"locations":[{"id":"pmh:oai:arXiv.org:2003.12206","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2003.12206","pdf_url":"https://arxiv.org/pdf/2003.12206","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:HAL:hal-03341560v1","doi":null,"is_oa":true,"landing_page_url":"https://hal.science/hal-03341560","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4405252976","host_organization_name":"Centre pour la Communication Scientifique Directe","host_organization_lineage":["https://openalex.org/I4405252976"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Journal of Machine Learning Research, 2021, 22","raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:oai:papyrus.bib.umontreal.ca:1866/25785","doi":null,"is_oa":true,"landing_page_url":"http://hdl.handle.net/1866/25785","pdf_url":null,"source":{"id":"https://openalex.org/S4306402422","display_name":"Papyrus : Institutional Repository (Universit\u00e9 de Montr\u00e9al)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I70931966","host_organization_name":"Universit\u00e9 de Montr\u00e9al","host_organization_lineage":["https://openalex.org/I70931966"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"journal article"},{"id":"pmh:oai:umontreal.scholaris.ca:1866/25785","doi":null,"is_oa":true,"landing_page_url":"http://hdl.handle.net/1866/25785","pdf_url":null,"source":{"id":"https://openalex.org/S4306402422","display_name":"Papyrus : Institutional Repository (Universit\u00e9 de Montr\u00e9al)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I70931966","host_organization_name":"Universit\u00e9 de Montr\u00e9al","host_organization_lineage":["https://openalex.org/I70931966"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"journal article"},{"id":"doi:10.48550/arxiv.2003.12206","doi":"https://doi.org/10.48550/arxiv.2003.12206","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2003.12206","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2003.12206","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2003.12206","pdf_url":"https://arxiv.org/pdf/2003.12206","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3192401439.pdf","grobid_xml":"https://content.openalex.org/works/W3192401439.grobid-xml"},"referenced_works_count":25,"referenced_works":["https://openalex.org/W1767470961","https://openalex.org/W2041179993","https://openalex.org/W2071612615","https://openalex.org/W2108924730","https://openalex.org/W2114690085","https://openalex.org/W2161694107","https://openalex.org/W2215394013","https://openalex.org/W2257979135","https://openalex.org/W2406493898","https://openalex.org/W2496371950","https://openalex.org/W2570760970","https://openalex.org/W2739255396","https://openalex.org/W2754517384","https://openalex.org/W2768599997","https://openalex.org/W2788767722","https://openalex.org/W2896457183","https://openalex.org/W2937186836","https://openalex.org/W2940661136","https://openalex.org/W2946506331","https://openalex.org/W2963341956","https://openalex.org/W2963748792","https://openalex.org/W2970043916","https://openalex.org/W2974231756","https://openalex.org/W3011700633","https://openalex.org/W4225656869"],"related_works":["https://openalex.org/W4382363179","https://openalex.org/W7214956177","https://openalex.org/W4409415146","https://openalex.org/W4389771368","https://openalex.org/W3200758773","https://openalex.org/W4407124756","https://openalex.org/W7115741582","https://openalex.org/W7165548596","https://openalex.org/W4402673633","https://openalex.org/W4205249749"],"abstract_inverted_index":{"One":[0],"of":[1,49,84,149,157,168],"the":[2,34,47,66,82,95,102,118,121,147,150,158],"challenges":[3],"in":[4,28,108],"machine":[5,109,130],"learning":[6,131],"research":[7,50,107],"is":[8,22,41,53],"to":[9,45,58,69,77,116,182],"ensure":[10],"that":[11,21],"presented":[12,27],"and":[13,18,37,61,74,128,146],"published":[14],"results":[15,25],"are":[16],"sound":[17],"reliable.":[19],"Reproducibility,":[20],"obtaining":[23],"similar":[24],"as":[26,155,175,177],"a":[29,42,112,138,142],"paper":[30,159],"or":[31],"talk,":[32],"using":[33],"same":[35],"code":[36,139],"data":[38],"(when":[39],"available),":[40],"necessary":[43],"step":[44,57],"verify":[46],"reliability":[48],"findings.":[51],"Reproducibility":[52,79,153],"also":[54,80],"an":[55],"important":[56],"promote":[59],"open":[60],"accessible":[62],"research,":[63],"thereby":[64],"allowing":[65],"scientific":[67],"community":[68,122],"quickly":[70],"integrate":[71],"new":[72],"findings":[73],"convert":[75],"ideas":[76],"practice.":[78],"promotes":[81],"use":[83],"robust":[85],"experimental":[86],"workflows,":[87],"which":[88],"potentially":[89],"reduce":[90],"unintentional":[91],"errors.":[92],"In":[93,162],"2019,":[94],"Neural":[96],"Information":[97],"Processing":[98],"Systems":[99],"(NeurIPS)":[100],"conference,":[101],"premier":[103],"international":[104],"conference":[105],"for":[106,123],"learning,":[110],"introduced":[111],"reproducibility":[113,144],"program,":[114],"designed":[115],"improve":[117],"standards":[119],"across":[120],"how":[124,171],"we":[125,165,179],"conduct,":[126],"communicate,":[127],"evaluate":[129],"research.":[132],"The":[133],"program":[134],"contained":[135],"three":[136],"components:":[137],"submission":[140,160],"policy,":[141],"community-wide":[143],"challenge,":[145],"inclusion":[148],"Machine":[151],"Learning":[152],"checklist":[154],"part":[156],"process.":[161],"this":[163,185],"paper,":[164],"describe":[166],"each":[167],"these":[169],"components,":[170],"it":[172],"was":[173],"deployed,":[174],"well":[176],"what":[178],"were":[180],"able":[181],"learn":[183],"from":[184],"initiative.":[186]},"counts_by_year":[{"year":2026,"cited_by_count":37},{"year":2025,"cited_by_count":62},{"year":2024,"cited_by_count":52},{"year":2023,"cited_by_count":59},{"year":2022,"cited_by_count":46},{"year":2021,"cited_by_count":46},{"year":2020,"cited_by_count":15}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2025-10-10T00:00:00"}