{"id":"https://openalex.org/W2880383342","doi":"https://doi.org/10.24963/ijcai.2018/851","title":"TuringBox: An Experimental Platform for the Evaluation of AI Systems","display_name":"TuringBox: An Experimental Platform for the Evaluation of AI Systems","publication_year":2018,"publication_date":"2018-07-01","ids":{"openalex":"https://openalex.org/W2880383342","doi":"https://doi.org/10.24963/ijcai.2018/851","mag":"2880383342"},"language":"en","primary_location":{"id":"doi:10.24963/ijcai.2018/851","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2018/851","pdf_url":"https://www.ijcai.org/proceedings/2018/0851.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twenty-Seventh International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.ijcai.org/proceedings/2018/0851.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5076651767","display_name":"Ziv Epstein","orcid":"https://orcid.org/0000-0002-5831-5756"},"institutions":[{"id":"https://openalex.org/I4210142372","display_name":"Human Media","ror":"https://ror.org/04072nk43","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210142372"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ziv Epstein","raw_affiliation_strings":["MIT Media Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT Media Lab","institution_ids":["https://openalex.org/I4210142372"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050551131","display_name":"Blakeley H. Payne","orcid":"https://orcid.org/0000-0003-1852-5350"},"institutions":[{"id":"https://openalex.org/I4210142372","display_name":"Human Media","ror":"https://ror.org/04072nk43","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210142372"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Blakeley H. Payne","raw_affiliation_strings":["MIT Media Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT Media Lab","institution_ids":["https://openalex.org/I4210142372"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020734965","display_name":"Judy Hanwen Shen","orcid":"https://orcid.org/0000-0002-7864-5242"},"institutions":[{"id":"https://openalex.org/I4210142372","display_name":"Human Media","ror":"https://ror.org/04072nk43","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210142372"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Judy Hanwen Shen","raw_affiliation_strings":["MIT Media Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT Media Lab","institution_ids":["https://openalex.org/I4210142372"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031853735","display_name":"Casey Jisoo Hong","orcid":null},"institutions":[{"id":"https://openalex.org/I4210142372","display_name":"Human Media","ror":"https://ror.org/04072nk43","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210142372"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Casey Jisoo Hong","raw_affiliation_strings":["MIT Media Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT Media Lab","institution_ids":["https://openalex.org/I4210142372"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018235085","display_name":"Bjarke Felbo","orcid":null},"institutions":[{"id":"https://openalex.org/I4210142372","display_name":"Human Media","ror":"https://ror.org/04072nk43","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210142372"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bjarke Felbo","raw_affiliation_strings":["MIT Media Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT Media Lab","institution_ids":["https://openalex.org/I4210142372"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020152678","display_name":"Abhimanyu Dubey","orcid":null},"institutions":[{"id":"https://openalex.org/I4210142372","display_name":"Human Media","ror":"https://ror.org/04072nk43","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210142372"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Abhimanyu Dubey","raw_affiliation_strings":["MIT Media Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT Media Lab","institution_ids":["https://openalex.org/I4210142372"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042355795","display_name":"Matthew Groh","orcid":"https://orcid.org/0000-0002-9029-0157"},"institutions":[{"id":"https://openalex.org/I4210142372","display_name":"Human Media","ror":"https://ror.org/04072nk43","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210142372"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Matthew Groh","raw_affiliation_strings":["MIT Media Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT Media Lab","institution_ids":["https://openalex.org/I4210142372"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085139454","display_name":"Nick Obradovich","orcid":"https://orcid.org/0000-0003-1127-2231"},"institutions":[{"id":"https://openalex.org/I4210142372","display_name":"Human Media","ror":"https://ror.org/04072nk43","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210142372"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nick Obradovich","raw_affiliation_strings":["MIT Media Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT Media Lab","institution_ids":["https://openalex.org/I4210142372"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017095669","display_name":"Manuel Cebri\u00e1n","orcid":"https://orcid.org/0000-0002-3681-7982"},"institutions":[{"id":"https://openalex.org/I4210142372","display_name":"Human Media","ror":"https://ror.org/04072nk43","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210142372"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Manuel Cebrian","raw_affiliation_strings":["MIT Media Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT Media Lab","institution_ids":["https://openalex.org/I4210142372"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5009232244","display_name":"Iyad Rahwan","orcid":"https://orcid.org/0000-0002-1796-4303"},"institutions":[{"id":"https://openalex.org/I4210142372","display_name":"Human Media","ror":"https://ror.org/04072nk43","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210142372"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Iyad Rahwan","raw_affiliation_strings":["MIT Media Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT Media Lab","institution_ids":["https://openalex.org/I4210142372"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I4210142372"],"apc_list":null,"apc_paid":null,"fwci":1.9058,"has_fulltext":false,"cited_by_count":18,"citation_normalized_percentile":{"value":0.85467573,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"5826","last_page":"5828"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12002","display_name":"Computability, Logic, AI Algorithms","score":0.994700014591217,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12002","display_name":"Computability, Logic, AI Algorithms","score":0.994700014591217,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9941999912261963,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9915000200271606,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8062522411346436},{"id":"https://openalex.org/keywords/upload","display_name":"Upload","score":0.7321261763572693},{"id":"https://openalex.org/keywords/architecture","display_name":"Architecture","score":0.5560250282287598},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4770295321941376},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.4605158567428589},{"id":"https://openalex.org/keywords/audit","display_name":"Audit","score":0.42877617478370667},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.38126906752586365},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.3346017599105835},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.1876448690891266}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8062522411346436},{"id":"https://openalex.org/C71901391","wikidata":"https://www.wikidata.org/wiki/Q7126699","display_name":"Upload","level":2,"score":0.7321261763572693},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.5560250282287598},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4770295321941376},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.4605158567428589},{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.42877617478370667},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.38126906752586365},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3346017599105835},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.1876448690891266},{"id":"https://openalex.org/C153349607","wikidata":"https://www.wikidata.org/wiki/Q36649","display_name":"Visual arts","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.24963/ijcai.2018/851","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2018/851","pdf_url":"https://www.ijcai.org/proceedings/2018/0851.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twenty-Seventh International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},{"id":"pmh:oai:pure.mpg.de:item_3020343","is_oa":true,"landing_page_url":"http://hdl.handle.net/21.11116/0000-0003-1A4C-C","pdf_url":null,"source":{"id":"https://openalex.org/S4306400654","display_name":"MPG.PuRe (Max Planck Society)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I149899117","host_organization_name":"Max Planck Society","host_organization_lineage":["https://openalex.org/I149899117"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Proceedings of the Twenty-Seventh International Joint Conference on Artificial Intelligence (IJCAI-18)","raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":{"id":"doi:10.24963/ijcai.2018/851","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2018/851","pdf_url":"https://www.ijcai.org/proceedings/2018/0851.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twenty-Seventh International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2880383342.pdf","grobid_xml":"https://content.openalex.org/works/W2880383342.grobid-xml"},"referenced_works_count":12,"referenced_works":["https://openalex.org/W2006447892","https://openalex.org/W2095932468","https://openalex.org/W2161392727","https://openalex.org/W2483215953","https://openalex.org/W2573660794","https://openalex.org/W2726198599","https://openalex.org/W2764072425","https://openalex.org/W2787225861","https://openalex.org/W2788481061","https://openalex.org/W2950018712","https://openalex.org/W2962787423","https://openalex.org/W2964235839"],"related_works":["https://openalex.org/W2944823289","https://openalex.org/W3037018281","https://openalex.org/W2003209439","https://openalex.org/W4321854979","https://openalex.org/W2358319515","https://openalex.org/W2972592048","https://openalex.org/W4312214821","https://openalex.org/W2497626292","https://openalex.org/W2390344072","https://openalex.org/W3089066832"],"abstract_inverted_index":{"We":[0,50],"introduce":[1],"TuringBox,":[2],"a":[3,56],"platform":[4],"to":[5,24,42],"democratize":[6],"the":[7,15,31,46,52,68],"study":[8],"of":[9,14,48,54,64],"AI.":[10],"On":[11,30],"one":[12],"side":[13],"platform,":[16,57],"AI":[17,34],"contributors":[18],"upload":[19],"existing":[20],"and":[21,37,44,58],"novel":[22],"algorithms":[23],"be":[25],"studied":[26],"scientifically":[27],"by":[28],"others.":[29],"other":[32],"side,":[33],"examiners":[35],"develop":[36],"post":[38],"machine":[39],"intelligence":[40],"tasks":[41],"evaluate":[43],"characterize":[45],"outputs":[47],"algorithms.":[49],"outline":[51],"architecture":[53],"such":[55],"describe":[59],"two":[60],"interactive":[61],"case":[62],"studies":[63],"algorithmic":[65],"auditing":[66],"on":[67],"platform.":[69]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":4},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":5},{"year":2020,"cited_by_count":1}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}