{"id":"https://openalex.org/W2793962634","doi":"https://doi.org/10.1109/access.2018.2800016","title":"Performance Evaluation Gaps in a Real-Time Strategy Game Between Human and Artificial Intelligence Players","display_name":"Performance Evaluation Gaps in a Real-Time Strategy Game Between Human and Artificial Intelligence Players","publication_year":2018,"publication_date":"2018-01-01","ids":{"openalex":"https://openalex.org/W2793962634","doi":"https://doi.org/10.1109/access.2018.2800016","mag":"2793962634"},"language":"en","primary_location":{"id":"doi:10.1109/access.2018.2800016","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2018.2800016","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1109/access.2018.2800016","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5067338383","display_name":"Man-Je Kim","orcid":"https://orcid.org/0000-0002-6402-2861"},"institutions":[{"id":"https://openalex.org/I28777354","display_name":"Sejong University","ror":"https://ror.org/00aft1q37","country_code":"KR","type":"education","lineage":["https://openalex.org/I28777354"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Man-Je Kim","raw_affiliation_strings":["Department of Computer Engineering, Sejong University, Seoul, South Korea","ORCiD"],"raw_orcid":"https://orcid.org/0000-0002-6402-2861","affiliations":[{"raw_affiliation_string":"Department of Computer Engineering, Sejong University, Seoul, South Korea","institution_ids":["https://openalex.org/I28777354"]},{"raw_affiliation_string":"ORCiD","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076055880","display_name":"Kyung-Joong Kim","orcid":"https://orcid.org/0000-0002-7732-0817"},"institutions":[{"id":"https://openalex.org/I28777354","display_name":"Sejong University","ror":"https://ror.org/00aft1q37","country_code":"KR","type":"education","lineage":["https://openalex.org/I28777354"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Kyung-Joong Kim","raw_affiliation_strings":["Department of Computer Engineering, Sejong University, Seoul, South Korea","ORCiD"],"raw_orcid":"https://orcid.org/0000-0002-7732-0817","affiliations":[{"raw_affiliation_string":"Department of Computer Engineering, Sejong University, Seoul, South Korea","institution_ids":["https://openalex.org/I28777354"]},{"raw_affiliation_string":"ORCiD","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024970503","display_name":"SeungJun Kim","orcid":"https://orcid.org/0000-0003-0470-2483"},"institutions":[{"id":"https://openalex.org/I39534123","display_name":"Gwangju Institute of Science and Technology","ror":"https://ror.org/024kbgz78","country_code":"KR","type":"education","lineage":["https://openalex.org/I39534123"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Seungjun Kim","raw_affiliation_strings":["Institute of Integrated Technology, Gwangju Institute of Science and Technology, Gwangju, South Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Integrated Technology, Gwangju Institute of Science and Technology, Gwangju, South Korea","institution_ids":["https://openalex.org/I39534123"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5032134965","display_name":"Anind K. Dey","orcid":"https://orcid.org/0000-0002-3004-0770"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Anind K. Dey","raw_affiliation_strings":["Human-Computer Interaction Institute, Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Human-Computer Interaction Institute, Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":2.4569,"has_fulltext":false,"cited_by_count":28,"citation_normalized_percentile":{"value":0.9136664,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":"6","issue":null,"first_page":"13575","last_page":"13586"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11574","display_name":"Artificial Intelligence in Games","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11574","display_name":"Artificial Intelligence in Games","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10731","display_name":"Educational Games and Gamification","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/3204","display_name":"Developmental and Educational Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11197","display_name":"Digital Games and Media","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/tournament","display_name":"Tournament","score":0.8786343336105347},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7269101142883301},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.657733142375946},{"id":"https://openalex.org/keywords/competitor-analysis","display_name":"Competitor analysis","score":0.624421238899231},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6231213808059692},{"id":"https://openalex.org/keywords/human-intelligence","display_name":"Human intelligence","score":0.587298572063446},{"id":"https://openalex.org/keywords/applications-of-artificial-intelligence","display_name":"Applications of artificial intelligence","score":0.5180006623268127},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5098787546157837},{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.4159894585609436},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.32022085785865784},{"id":"https://openalex.org/keywords/marketing","display_name":"Marketing","score":0.09755584597587585},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.06920105218887329}],"concepts":[{"id":"https://openalex.org/C136975688","wikidata":"https://www.wikidata.org/wiki/Q1320634","display_name":"Tournament","level":2,"score":0.8786343336105347},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7269101142883301},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.657733142375946},{"id":"https://openalex.org/C127576917","wikidata":"https://www.wikidata.org/wiki/Q624630","display_name":"Competitor analysis","level":2,"score":0.624421238899231},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6231213808059692},{"id":"https://openalex.org/C105409693","wikidata":"https://www.wikidata.org/wiki/Q5937824","display_name":"Human intelligence","level":2,"score":0.587298572063446},{"id":"https://openalex.org/C157170001","wikidata":"https://www.wikidata.org/wiki/Q4781507","display_name":"Applications of artificial intelligence","level":2,"score":0.5180006623268127},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5098787546157837},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.4159894585609436},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.32022085785865784},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.09755584597587585},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.06920105218887329},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/access.2018.2800016","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2018.2800016","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:f3bc54e301244eda95c1b1dc45176b78","is_oa":true,"landing_page_url":"https://doaj.org/article/f3bc54e301244eda95c1b1dc45176b78","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 6, Pp 13575-13586 (2018)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1109/access.2018.2800016","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2018.2800016","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G5687761049","display_name":null,"funder_award_id":"2017R1A2B4002164","funder_id":"https://openalex.org/F4320322120","funder_display_name":"National Research Foundation of Korea"}],"funders":[{"id":"https://openalex.org/F4320322120","display_name":"National Research Foundation of Korea","ror":"https://ror.org/013aysd81"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":17,"referenced_works":["https://openalex.org/W1487104801","https://openalex.org/W1572584819","https://openalex.org/W1660754132","https://openalex.org/W1911071008","https://openalex.org/W1983444181","https://openalex.org/W2031734221","https://openalex.org/W2066998295","https://openalex.org/W2132266610","https://openalex.org/W2151162056","https://openalex.org/W2160765669","https://openalex.org/W2183966306","https://openalex.org/W2516941566","https://openalex.org/W2569196563","https://openalex.org/W4232217679","https://openalex.org/W4299335686","https://openalex.org/W6686581360","https://openalex.org/W6726280463"],"related_works":["https://openalex.org/W2951933385","https://openalex.org/W190274399","https://openalex.org/W3121108812","https://openalex.org/W3139434641","https://openalex.org/W3151498429","https://openalex.org/W2161148151","https://openalex.org/W4386271515","https://openalex.org/W3044189763","https://openalex.org/W2997038949","https://openalex.org/W4313891971"],"abstract_inverted_index":{"Since":[0],"2010,":[1],"annual":[2],"StarCraft":[3,62,123],"artificial":[4],"intelligence":[5],"(AI)":[6],"competitions":[7,86],"have":[8],"promoted":[9],"the":[10,75,82,92,101,120,127,151,156,162,168,195],"development":[11],"of":[12,35,85,95,122,159,164,197],"successful":[13],"AI":[14,24,53,78,96,124,160,165,183,189,202],"players":[15,25,63,79,97,125],"for":[16,146],"complex":[17],"real-time":[18],"strategy":[19],"games.":[20],"In":[21,55,108],"these":[22],"competitions,":[23],"are":[26,143],"ranked":[27],"based":[28],"on":[29],"their":[30,201],"win":[31],"ratio":[32],"over":[33,126],"thousands":[34],"head-to-head":[36],"matches.":[37],"Although":[38],"simple":[39],"and":[40,69,105,167,173,188],"easily":[41],"implemented,":[42],"this":[43,56,179],"evaluation":[44,104,158],"scheme":[45],"may":[46],"less":[47],"adequately":[48],"help":[49],"develop":[50],"more":[51],"human-competitive":[52],"players.":[54],"paper,":[57],"we":[58],"recruited":[59],"45":[60],"human":[61,93,112,157,171,198],"at":[64],"different":[65],"expertise":[66],"levels":[67],"(expert/medium/novice)":[68],"asked":[70],"them":[71],"to":[72,138,153,192],"play":[73],"against":[74],"18":[76],"top":[77],"selected":[80],"from":[81,100,110,178],"five":[83],"years":[84],"(2011-2015).":[87],"The":[88,176],"results":[89],"show":[90],"that":[91,133,142],"evaluations":[94],"differ":[98],"substantially":[99],"current":[102],"standard":[103],"ranking":[106],"method.":[107],"fact,":[109],"a":[111],"standpoint,":[113],"there":[114],"has":[115],"been":[116],"little":[117],"progress":[118],"in":[119,185],"quality":[121],"years.":[128],"It":[129],"is":[130,150],"even":[131],"possible":[132],"AI-only":[134],"tournaments":[135],"can":[136,181],"lead":[137],"AIs":[139],"being":[140],"created":[141],"unacceptable":[144],"competitors":[145],"humans.":[147],"This":[148],"paper":[149,180],"first":[152],"systematically":[154],"explore":[155],"players,":[161,166],"evolution":[163],"differences":[169],"between":[170],"perception":[172],"tournament-based":[174],"evaluations.":[175],"discoveries":[177],"support":[182],"developers":[184],"game":[186],"companies":[187],"tournament":[190],"organizers":[191],"better":[193],"incorporate":[194],"perspective":[196],"users":[199],"into":[200],"systems.":[203]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":6},{"year":2021,"cited_by_count":9},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":3},{"year":2018,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}