{"id":"https://openalex.org/W7162783512","doi":"https://doi.org/10.48550/arxiv.2605.29786","title":"Croissant Tasks: A Metadata Format for Reproducible Machine Learning Evaluations","display_name":"Croissant Tasks: A Metadata Format for Reproducible Machine Learning Evaluations","publication_year":2026,"publication_date":"2026-05-28","ids":{"openalex":"https://openalex.org/W7162783512","doi":"https://doi.org/10.48550/arxiv.2605.29786"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.29786","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.29786","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.29786","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103212061","display_name":"Omar Benjelloun","orcid":"https://orcid.org/0000-0002-4173-7709"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Benjelloun, Omar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137326927","display_name":"Leonardo Martins Bianco","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bianco, Leonardo Martins","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137324392","display_name":"Isabelle Guyon","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guyon, Isabelle","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093047586","display_name":"Thanh Gia Hieu Khuong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Khuong, Thanh Gia Hieu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075108760","display_name":"Jonathan Lebensold","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lebensold, Jonathan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137327286","display_name":"Sebastian Lobentanzer","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lobentanzer, Sebastian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137357356","display_name":"Luis Oala","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Oala, Luis","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093047587","display_name":"Benedictus Kent Rachmat","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rachmat, Benedictus Kent","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137311829","display_name":"Ihsan Ullah","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ullah, Ihsan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119268198","display_name":"Peyman Vahidi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Vahidi, Peyman","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5137363831","display_name":"Joaquin Vanschoren","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Vanschoren, Joaquin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.6625000238418579,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.6625000238418579,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.0706000030040741,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.05739999935030937,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/metadata","display_name":"Metadata","score":0.7771000266075134},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.5284000039100647},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.4609000086784363},{"id":"https://openalex.org/keywords/implementation","display_name":"Implementation","score":0.44699999690055847},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.36469998955726624},{"id":"https://openalex.org/keywords/data-format","display_name":"Data format","score":0.3197999894618988},{"id":"https://openalex.org/keywords/software-quality","display_name":"Software quality","score":0.31200000643730164}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8095999956130981},{"id":"https://openalex.org/C93518851","wikidata":"https://www.wikidata.org/wiki/Q180160","display_name":"Metadata","level":2,"score":0.7771000266075134},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.5284000039100647},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.5242999792098999},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4609000086784363},{"id":"https://openalex.org/C26713055","wikidata":"https://www.wikidata.org/wiki/Q245962","display_name":"Implementation","level":2,"score":0.44699999690055847},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38909998536109924},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.38440001010894775},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.36469998955726624},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.32330000400543213},{"id":"https://openalex.org/C2985331491","wikidata":"https://www.wikidata.org/wiki/Q5227298","display_name":"Data format","level":2,"score":0.3197999894618988},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.31200000643730164},{"id":"https://openalex.org/C2776303644","wikidata":"https://www.wikidata.org/wiki/Q1020499","display_name":"Interfacing","level":2,"score":0.29809999465942383},{"id":"https://openalex.org/C205606062","wikidata":"https://www.wikidata.org/wiki/Q5249645","display_name":"Decoupling (probability)","level":2,"score":0.29739999771118164},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.29089999198913574},{"id":"https://openalex.org/C20136886","wikidata":"https://www.wikidata.org/wiki/Q749647","display_name":"Interoperability","level":2,"score":0.28060001134872437},{"id":"https://openalex.org/C519991488","wikidata":"https://www.wikidata.org/wiki/Q28865","display_name":"Python (programming language)","level":2,"score":0.26330000162124634},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.26109999418258667},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.25850000977516174}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.29786","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.29786","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.29786","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.29786","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Reproducibility":[0],"is":[1],"fundamental":[2],"to":[3,40,115],"the":[4,82],"scientific":[5],"method,":[6],"yet":[7],"remains":[8],"a":[9,49,128],"critical":[10],"challenge":[11],"in":[12,136],"machine":[13,137],"learning.":[14,138],"Contributing":[15],"factors":[16],"include":[17],"underspecified":[18],"execution":[19],"details":[20,58],"and":[21,30,38,104,133],"brittle":[22,75],"software":[23],"environments.":[24],"Human-centric":[25],"remedies,":[26],"such":[27],"as":[28,127],"checklists":[29],"manual":[31],"verification,":[32],"help":[33],"but":[34],"require":[35],"intensive":[36],"effort":[37],"fail":[39],"scale.":[41],"To":[42],"address":[43],"this,":[44],"we":[45],"introduce":[46],"Croissant":[47,83],"Tasks:":[48],"declarative,":[50],"machine-actionable":[51],"metadata":[52],"format":[53,63,126],"that":[54,97],"abstracts":[55],"low-level":[56],"implementation":[57],"into":[59,101],"high-level":[60],"specifications.":[61],"This":[62],"enables":[64],"conceptual":[65,134],"reproducibility:":[66],"verifying":[67],"claims":[68],"via":[69],"independent,":[70],"agent-generated":[71],"implementations":[72],"rather":[73],"than":[74],"source":[76],"code":[77],"replication.":[78],"We":[79,123],"contribute:":[80],"(1)":[81],"Tasks":[84],"specification,":[85],"formally":[86],"decoupling":[87],"task":[88],"problem":[89],"from":[90,121],"solution;":[91],"(2)":[92],"an":[93],"automated":[94,132],"LLM":[95],"pipeline":[96],"retrofits":[98],"existing":[99],"benchmarks":[100],"this":[102,125],"format;":[103],"(3)":[105],"empirical":[106],"validation":[107],"showing":[108],"autonomous":[109],"agents":[110],"can":[111],"ingest":[112],"these":[113],"specifications":[114],"generate":[116],"functional,":[117],"accurate":[118],"reproduction":[119],"pipelines":[120],"scratch.":[122],"envision":[124],"new":[129],"foundation":[130],"for":[131],"reproducibility":[135]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-30T00:00:00"}