{"id":"https://openalex.org/W4390960299","doi":"https://doi.org/10.48550/arxiv.2401.13693","title":"Challenge design roadmap","display_name":"Challenge design roadmap","publication_year":2024,"publication_date":"2024-01-15","ids":{"openalex":"https://openalex.org/W4390960299","doi":"https://doi.org/10.48550/arxiv.2401.13693"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2401.13693","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2401.13693","pdf_url":"https://arxiv.org/pdf/2401.13693","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2401.13693","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5078959623","display_name":"Hugo Jair Escalante Balderas","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I39824353","display_name":"National Institute of Astrophysics, Optics and Electronics","ror":"https://ror.org/00bpmmc63","country_code":"MX","type":"facility","lineage":["https://openalex.org/I39824353"]},{"id":"https://openalex.org/I4387152856","display_name":"Laboratoire Interdisciplinaire des Sciences du Num\u00e9rique","ror":"https://ror.org/00rd81916","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I277688954","https://openalex.org/I4387152856"]},{"id":"https://openalex.org/I4405259997","display_name":"TAU: IA : vers des probl\u00e8mes mieux pos\u00e9s","ror":"https://ror.org/031y5kf09","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1326498283","https://openalex.org/I277688954","https://openalex.org/I4210126360","https://openalex.org/I4387152856","https://openalex.org/I4405256294","https://openalex.org/I4405259997"]}],"countries":["FR","MX"],"is_corresponding":false,"raw_author_name":"Balderas, Hugo Jair Escalante","raw_affiliation_strings":["LISN, TAU","INAOE - Instituto Nacional de Astrof\u00edsica, \u00d3ptica y Electr\u00f3nica (Luis Enrique Erro 1, Sta Mar\u00eda Tonanzintla, 72840 Puebla, Pue., Mexique - Mexico)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LISN, TAU","institution_ids":["https://openalex.org/I4387152856","https://openalex.org/I4405259997"]},{"raw_affiliation_string":"INAOE - Instituto Nacional de Astrof\u00edsica, \u00d3ptica y Electr\u00f3nica (Luis Enrique Erro 1, Sta Mar\u00eda Tonanzintla, 72840 Puebla, Pue., Mexique - Mexico)","institution_ids":["https://openalex.org/I39824353"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006360335","display_name":"Isabelle Guyon","orcid":"https://orcid.org/0000-0002-9266-1783","observed_orcids":["https://orcid.org/0000-0002-9266-1783"]},"institutions":[{"id":"https://openalex.org/I277688954","display_name":"Universit\u00e9 Paris-Saclay","ror":"https://ror.org/03xjwb503","country_code":"FR","type":"education","lineage":["https://openalex.org/I277688954"]},{"id":"https://openalex.org/I4387152856","display_name":"Laboratoire Interdisciplinaire des Sciences du Num\u00e9rique","ror":"https://ror.org/00rd81916","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I277688954","https://openalex.org/I4387152856"]},{"id":"https://openalex.org/I4405259997","display_name":"TAU: IA : vers des probl\u00e8mes mieux pos\u00e9s","ror":"https://ror.org/031y5kf09","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1326498283","https://openalex.org/I277688954","https://openalex.org/I4210126360","https://openalex.org/I4387152856","https://openalex.org/I4405256294","https://openalex.org/I4405259997"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Guyon, Isabelle","raw_affiliation_strings":["LISN, TAU","LISN - Laboratoire Interdisciplinaire des Sciences du Num\u00e9rique (Campus Universitaire b\u00e2t 507\r\nRue du Belv\u00e9d\u00e8re\r\n91405 Orsay cedex\r\net \r\nCampus Universitaire b\u00e2t 640\r\n1 rue Raimond Castaing\r\n91190 Gif-sur-Yvette - France)","TAU - TAckling the Underspecified (Universit\u00e9 Paris Saclay\r\nB\u00e2timent 660 - Shannon ; \r\n91405 Orsay Cedex - France)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LISN, TAU","institution_ids":["https://openalex.org/I4387152856","https://openalex.org/I4405259997"]},{"raw_affiliation_string":"LISN - Laboratoire Interdisciplinaire des Sciences du Num\u00e9rique (Campus Universitaire b\u00e2t 507\r\nRue du Belv\u00e9d\u00e8re\r\n91405 Orsay cedex\r\net \r\nCampus Universitaire b\u00e2t 640\r\n1 rue Raimond Castaing\r\n91190 Gif-sur-Yvette - France)","institution_ids":["https://openalex.org/I4387152856"]},{"raw_affiliation_string":"TAU - TAckling the Underspecified (Universit\u00e9 Paris Saclay\r\nB\u00e2timent 660 - Shannon ; \r\n91405 Orsay Cedex - France)","institution_ids":["https://openalex.org/I277688954","https://openalex.org/I4405259997"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012276996","display_name":"Addison Howard","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I16391192","display_name":"Tel Aviv University","ror":"https://ror.org/04mhzgx49","country_code":"IL","type":"education","lineage":["https://openalex.org/I16391192"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Howard, Addison","raw_affiliation_strings":["TAU"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"TAU","institution_ids":["https://openalex.org/I16391192"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033121212","display_name":"Walter Reade","orcid":"https://orcid.org/0000-0002-9215-700X","observed_orcids":["https://orcid.org/0000-0002-9215-700X"]},"institutions":[{"id":"https://openalex.org/I16391192","display_name":"Tel Aviv University","ror":"https://ror.org/04mhzgx49","country_code":"IL","type":"education","lineage":["https://openalex.org/I16391192"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Reade, Walter","raw_affiliation_strings":["TAU"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"TAU","institution_ids":["https://openalex.org/I16391192"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5080573113","display_name":"S\u00e9bastien Treguer","orcid":"https://orcid.org/0000-0001-8733-9805","observed_orcids":["https://orcid.org/0000-0001-8733-9805"]},"institutions":[{"id":"https://openalex.org/I16391192","display_name":"Tel Aviv University","ror":"https://ror.org/04mhzgx49","country_code":"IL","type":"education","lineage":["https://openalex.org/I16391192"]},{"id":"https://openalex.org/I277688954","display_name":"Universit\u00e9 Paris-Saclay","ror":"https://ror.org/03xjwb503","country_code":"FR","type":"education","lineage":["https://openalex.org/I277688954"]},{"id":"https://openalex.org/I4405259997","display_name":"TAU: IA : vers des probl\u00e8mes mieux pos\u00e9s","ror":"https://ror.org/031y5kf09","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1326498283","https://openalex.org/I277688954","https://openalex.org/I4210126360","https://openalex.org/I4387152856","https://openalex.org/I4405256294","https://openalex.org/I4405259997"]}],"countries":["FR","IL"],"is_corresponding":false,"raw_author_name":"Treguer, Sebastien","raw_affiliation_strings":["TAU","TAU - TAckling the Underspecified (Universit\u00e9 Paris Saclay\r\nB\u00e2timent 660 - Shannon ; \r\n91405 Orsay Cedex - France)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"TAU","institution_ids":["https://openalex.org/I16391192"]},{"raw_affiliation_string":"TAU - TAckling the Underspecified (Universit\u00e9 Paris Saclay\r\nB\u00e2timent 660 - Shannon ; \r\n91405 Orsay Cedex - France)","institution_ids":["https://openalex.org/I277688954","https://openalex.org/I4405259997"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10731","display_name":"Educational Games and Gamification","score":0.20749999582767487,"subfield":{"id":"https://openalex.org/subfields/3204","display_name":"Developmental and Educational Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10731","display_name":"Educational Games and Gamification","score":0.20749999582767487,"subfield":{"id":"https://openalex.org/subfields/3204","display_name":"Developmental and Educational Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13280","display_name":"Biomedical and Engineering Education","score":0.16060000658035278,"subfield":{"id":"https://openalex.org/subfields/2204","display_name":"Biomedical Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.11379999667406082,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/challenge-design","display_name":"challenge design","score":0.8740000128746033},{"id":"https://openalex.org/keywords/competition-organization","display_name":"competition organization","score":0.4410000145435333},{"id":"https://openalex.org/keywords/game-rules","display_name":"game rules","score":0.7739999890327454},{"id":"https://openalex.org/keywords/serious-games","display_name":"serious games","score":0.7829999923706055},{"id":"https://openalex.org/keywords/peer-review","display_name":"peer review","score":0.9789999723434448},{"id":"https://openalex.org/keywords/kaggle","display_name":"Kaggle","score":0.5910000205039978},{"id":"https://openalex.org/keywords/tailors","display_name":"tailors","score":0.7360000014305115}],"concepts":[{"id":"https://openalex.org/C2776505523","wikidata":"https://www.wikidata.org/wiki/Q4785468","display_name":"Plan (archaeology)","level":2,"score":0.7064574956893921},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.663677453994751},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.6563952565193176},{"id":"https://openalex.org/C91306197","wikidata":"https://www.wikidata.org/wiki/Q45767","display_name":"Competition (biology)","level":2,"score":0.6438472867012024},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5705470442771912},{"id":"https://openalex.org/C90673727","wikidata":"https://www.wikidata.org/wiki/Q901718","display_name":"Product (mathematics)","level":2,"score":0.5461261868476868},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.45460978150367737},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.3363112807273865},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.3213387429714203},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.2563689947128296},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.17494168877601624},{"id":"https://openalex.org/C18903297","wikidata":"https://www.wikidata.org/wiki/Q7150","display_name":"Ecology","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"pmh:oai:arXiv.org:2401.13693","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2401.13693","pdf_url":"https://arxiv.org/pdf/2401.13693","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:HAL:hal-04333280v1","doi":null,"is_oa":true,"landing_page_url":"https://hal.science/hal-04333280","pdf_url":"https://hal.science/hal-04333280/document","source":{"id":"https://openalex.org/S4306402512","display_name":"HAL","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4405252976","host_organization_name":"Centre pour la Communication Scientifique Directe","host_organization_lineage":["https://openalex.org/I4405252976"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"AI Competitions and Benchmarks: The Science Behind the Contests","raw_type":"info:eu-repo/semantics/bookPart"},{"id":"pmh:oai:HAL:hal-04333280v2","doi":null,"is_oa":true,"landing_page_url":"https://hal.science/hal-04333280","pdf_url":"https://hal.science/hal-04333280v2/file/Chapter2.pdf","source":{"id":"https://openalex.org/S4306402512","display_name":"HAL","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4405252976","host_organization_name":"Centre pour la Communication Scientifique Directe","host_organization_lineage":["https://openalex.org/I4405252976"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"AI Competitions and Benchmarks: The Science Behind the Contests, In press","raw_type":"info:eu-repo/semantics/bookPart"},{"id":"doi:10.48550/arxiv.2401.13693","doi":"https://doi.org/10.48550/arxiv.2401.13693","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2401.13693","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article-journal"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2401.13693","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2401.13693","pdf_url":"https://arxiv.org/pdf/2401.13693","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G1045782842","display_name":"Artificial Intelligence for All","funder_award_id":"ANR-19-CHIA-0022","funder_id":"https://openalex.org/F4320320883","funder_display_name":"Agence Nationale de la Recherche"},{"id":"https://openalex.org/G3010527862","display_name":null,"funder_award_id":"ANR\u201019","funder_id":"https://openalex.org/F4320320883","funder_display_name":"Agence Nationale de la Recherche"}],"funders":[{"id":"https://openalex.org/F4320320883","display_name":"Agence Nationale de la Recherche","ror":"https://ror.org/00rbzpz17"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4390960299.pdf","grobid_xml":"https://content.openalex.org/works/W4390960299.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W203840240","https://openalex.org/W1123716344","https://openalex.org/W2271349231","https://openalex.org/W4239058122","https://openalex.org/W4406174084","https://openalex.org/W1989580853","https://openalex.org/W2613971546","https://openalex.org/W4243749047","https://openalex.org/W2042757477","https://openalex.org/W2473232573"],"abstract_inverted_index":{"Challenges":[0],"can":[1],"be":[2,107],"seen":[3],"as":[4,100,166,176,178],"a":[5,17,62,68,96,101,150,154],"type":[6],"of":[7,75,90,133,185],"game":[8,24,35],"that":[9,104],"motivates":[10],"participants":[11],"to":[12,66,84,109,115,129],"solve":[13],"serious":[14],"tasks.":[15],"As":[16],"result,":[18],"competition":[19,102],"organizers":[20,128],"must":[21],"develop":[22],"effective":[23],"rules.":[25],"However,":[26],"these":[27],"rules":[28],"have":[29],"multiple":[30],"objectives":[31,40],"beyond":[32],"making":[33,51],"the":[34,56,72,81,88,131,160,179,186],"enjoyable":[36],"for":[37,148,153],"participants.":[38,91],"These":[39],"may":[41],"include":[42],"solving":[43],"real-world":[44],"problems,":[45],"advancing":[46],"scientific":[47,52],"or":[48],"technical":[49],"areas,":[50],"discoveries,":[53],"and":[54,77,80,113,139,172],"educating":[55],"public.":[57],"In":[58],"many":[59],"ways,":[60],"creating":[61,149],"challenge":[63],"is":[64,83],"similar":[65],"launching":[67],"product.":[69],"It":[70],"requires":[71],"same":[73],"level":[74],"excitement":[76],"rigorous":[78],"testing,":[79],"goal":[82],"attract":[85],"''customers''":[86],"in":[87],"form":[89],"The":[92,156],"process":[93],"begins":[94],"with":[95],"solid":[97],"plan,":[98],"such":[99,165],"proposal":[103,181],"will":[105],"eventually":[106],"submitted":[108],"an":[110],"international":[111],"conference":[112],"subjected":[114],"peer":[116,119],"review.":[117],"Although":[118],"review":[120],"does":[121,126],"not":[122],"guarantee":[123],"quality,":[124],"it":[125],"force":[127],"consider":[130],"impact":[132],"their":[134],"challenge,":[135],"identify":[136],"potential":[137],"oversights,":[138],"generally":[140],"improve":[141],"its":[142],"quality.":[143],"This":[144],"chapter":[145],"provides":[146],"guidelines":[147,162],"strong":[151],"plan":[152],"challenge.":[155],"material":[157],"draws":[158],"on":[159],"preparation":[161],"from":[163],"organizations":[164],"Kaggle":[167],"1":[168],",":[169,175],"ChaLearn":[170],"2":[171],"Tailor":[173],"3":[174],"well":[177],"NeurIPS":[180],"template,":[182],"which":[183],"some":[184],"authors":[187],"contributed":[188],"to.":[189]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2024-01-18T00:00:00"}