{"id":"https://openalex.org/W2250031007","doi":"https://doi.org/10.1109/ssci.2015.89","title":"Temporal Difference Learning for the Game Tic-Tac-Toe 3D: Applying Structure to Neural Networks","display_name":"Temporal Difference Learning for the Game Tic-Tac-Toe 3D: Applying Structure to Neural Networks","alternate_titles":[],"publication_year":2015,"publication_date":"2015-12-01","ids":{"openalex":"https://openalex.org/W2250031007","doi":"https://doi.org/10.1109/ssci.2015.89","mag":"2250031007"},"language":"en","primary_location":{"id":"doi:10.1109/ssci.2015.89","doi":"https://doi.org/10.1109/ssci.2015.89","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ssci.2015.89","pdf_url":null,"source":{"id":"https://openalex.org/S4363604921","display_name":"IEEE Symposium Series on Computational Intelligence (SSCI)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2015 IEEE Symposium Series on Computational Intelligence","raw_type":"proceedings-article"},"parent_works":[],"lineage":["https://openalex.org/W2250031007"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://research.rug.nl/files/26826622/1570192495_Steeg_ADPRL_new.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5057962472","display_name":"Michiel van de Steeg","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I169381384","display_name":"University of Groningen","ror":"https://ror.org/012p63287","country_code":"NL","type":"education","lineage":["https://openalex.org/I169381384"]}],"countries":["NL"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Michiel van de Steeg","raw_affiliation_strings":["Institute of Artificial Intelligence and Cognitive Engineering, University of Groningen, Groningen, Groningen, NL"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Artificial Intelligence and Cognitive Engineering, University of Groningen, Groningen, Groningen, NL","institution_ids":["https://openalex.org/I169381384"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016131956","display_name":"M\u0103d\u0103lina M. Drugan","orcid":"https://orcid.org/0000-0001-9497-3134","observed_orcids":["https://orcid.org/0000-0001-9497-3134"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Madalina M. Drugan","raw_affiliation_strings":["Department of Computer Science Technical, University of Eindhoven, Eindhoven, The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science Technical, University of Eindhoven, Eindhoven, The Netherlands","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5060596453","display_name":"Marco Wiering","orcid":"https://orcid.org/0000-0003-4331-7537","observed_orcids":["https://orcid.org/0000-0003-4331-7537"]},"institutions":[{"id":"https://openalex.org/I169381384","display_name":"University of Groningen","ror":"https://ror.org/012p63287","country_code":"NL","type":"education","lineage":["https://openalex.org/I169381384"]}],"countries":["NL"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Marco Wiering","raw_affiliation_strings":["Institute of Artificial Intelligence and Cognitive Engineering, University of Groningen, The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Artificial Intelligence and Cognitive Engineering, University of Groningen, The Netherlands","institution_ids":["https://openalex.org/I169381384"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.3581,"has_fulltext":true,"cited_by_count":15,"reviewed_by_count":0,"citation_normalized_percentile":{"value":0.82649502,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":86,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"564","last_page":"570"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.8192999958992004,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.8192999958992004,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11574","display_name":"Artificial Intelligence in Games","score":0.17880000174045563,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.0008999999845400453,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/temporal-difference-learning","display_name":"temporal difference learning","score":0.9879999756813049},{"id":"https://openalex.org/keywords/deep-reinforcement-learning","display_name":"deep reinforcement learning","score":0.6629999876022339},{"id":"https://openalex.org/keywords/multilayer-perceptron","display_name":"multilayer perceptron","score":0.8190000057220459},{"id":"https://openalex.org/keywords/structured-neural-networks","display_name":"structured neural networks","score":0.7699999809265137},{"id":"https://openalex.org/keywords/sparse-connectivity","display_name":"sparse connectivity","score":0.5260000228881836},{"id":"https://openalex.org/keywords/value-function-approximation","display_name":"value function approximation","score":0.7730000019073486},{"id":"https://openalex.org/keywords/self-play","display_name":"self-play","score":0.6449999809265137},{"id":"https://openalex.org/keywords/board-games","display_name":"board games","score":0.7039999961853027},{"id":"https://openalex.org/keywords/pattern-detection","display_name":"pattern detection","score":0.5699999928474426},{"id":"https://openalex.org/keywords/hidden-layer-structure","display_name":"hidden layer structure","score":0.43700000643730164}],"concepts":[{"id":"https://openalex.org/C60908668","wikidata":"https://www.wikidata.org/wiki/Q690207","display_name":"Perceptron","level":3,"score":0.8728541135787964},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7250942587852478},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.6583647131919861},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6392409205436707},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6217706203460693},{"id":"https://openalex.org/C2779227376","wikidata":"https://www.wikidata.org/wiki/Q6505497","display_name":"Layer (electronics)","level":2,"score":0.4782547056674957},{"id":"https://openalex.org/C196340769","wikidata":"https://www.wikidata.org/wiki/Q7698910","display_name":"Temporal difference learning","level":3,"score":0.47199395298957825},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.46713781356811523},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.46558183431625366},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.43505632877349854},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.40073835849761963},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.1109/ssci.2015.89","doi":"https://doi.org/10.1109/ssci.2015.89","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ssci.2015.89","pdf_url":null,"source":{"id":"https://openalex.org/S4363604921","display_name":"IEEE Symposium Series on Computational Intelligence (SSCI)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2015 IEEE Symposium Series on Computational Intelligence","raw_type":"proceedings-article"},{"id":"pmh:oai:pure.rug.nl:openaire/8364aa86-11fc-4efc-a00e-afdee0e3e0e3","doi":null,"is_oa":true,"landing_page_url":"https://research.rug.nl/files/26826622/1570192495_Steeg_ADPRL_new.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400420","display_name":"University of Groningen research database (University of Groningen / Centre for Information Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I169381384","host_organization_name":"University of Groningen","host_organization_lineage":["https://openalex.org/I169381384"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Van De Steeg, M, Drugan, M M & Wiering, M 2016, Temporal difference learning for the game Tic-Tac-Toe 3D: Applying structure to neural networks. in Proceedings - 2015 IEEE Symposium Series on Computational Intelligence, SSCI 2015. Proceedings - 2015 IEEE Symposium Series on Computational Intelligence, SSCI 2015, Institute of Electrical and Electronics Engineers Inc., pp. 564-570. https://doi.org/10.1109/SSCI.2015.89","raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"pmh:oai:pure.tue.nl:openaire_cris_publications/949a873c-579d-4803-8ce7-360cd29be4cf","doi":null,"is_oa":false,"landing_page_url":"https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=7376662","pdf_url":null,"source":{"id":"https://openalex.org/S4406922641","display_name":"TU/e Research Portal","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"van de Steeg, M, Drugan, M M & Wiering, M 2015, Temporal difference learning for the game Tic-Tac-Toe 3D : applying structure to neural networks. in 2015 IEEE Symposium Series on Computational Intelligence, 7-10 December 2015, Cape Town, South Africa. Institute of Electrical and Electronics Engineers, Piscataway, pp. 564-570. https://doi.org/10.1109/SSCI.2015.89","raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"pmh:rug:oai:pure.rug.nl:publications/8364aa86-11fc-4efc-a00e-afdee0e3e0e3","doi":null,"is_oa":true,"landing_page_url":"https://research.rug.nl/en/publications/8364aa86-11fc-4efc-a00e-afdee0e3e0e3","pdf_url":null,"source":{"id":"https://openalex.org/S4306400420","display_name":"University of Groningen research database (University of Groningen / Centre for Information Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I169381384","host_organization_name":"University of Groningen","host_organization_lineage":["https://openalex.org/I169381384"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Proceedings - 2015 IEEE Symposium Series on Computational Intelligence, SSCI 2015, 564 - 570","raw_type":"info:eu-repo/semantics/conferencepaper"},{"id":"pmh:tue:oai:pure.tue.nl:publications/949a873c-579d-4803-8ce7-360cd29be4cf","doi":null,"is_oa":false,"landing_page_url":"https://research.tue.nl/nl/publications/949a873c-579d-4803-8ce7-360cd29be4cf","pdf_url":null,"source":{"id":"https://openalex.org/S4406922641","display_name":"TU/e Research Portal","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2015 IEEE Symposium Series on Computational Intelligence, 7-10 December 2015, Cape Town, South Africa, 564 - 570","raw_type":"info:eu-repo/semantics/conferencepaper"}],"best_oa_location":{"id":"pmh:oai:pure.rug.nl:openaire/8364aa86-11fc-4efc-a00e-afdee0e3e0e3","doi":null,"is_oa":true,"landing_page_url":"https://research.rug.nl/files/26826622/1570192495_Steeg_ADPRL_new.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400420","display_name":"University of Groningen research database (University of Groningen / Centre for Information Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I169381384","host_organization_name":"University of Groningen","host_organization_lineage":["https://openalex.org/I169381384"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Van De Steeg, M, Drugan, M M & Wiering, M 2016, Temporal difference learning for the game Tic-Tac-Toe 3D: Applying structure to neural networks. in Proceedings - 2015 IEEE Symposium Series on Computational Intelligence, SSCI 2015. Proceedings - 2015 IEEE Symposium Series on Computational Intelligence, SSCI 2015, Institute of Electrical and Electronics Engineers Inc., pp. 564-570. https://doi.org/10.1109/SSCI.2015.89","raw_type":"info:eu-repo/semantics/conferenceObject"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320313144","display_name":"ITEA","ror":"https://ror.org/04cfmv025"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":20,"referenced_works":["https://openalex.org/W613766092","https://openalex.org/W1515851193","https://openalex.org/W1601359266","https://openalex.org/W2028145673","https://openalex.org/W2041367235","https://openalex.org/W2076063813","https://openalex.org/W2099001564","https://openalex.org/W2100677568","https://openalex.org/W2101786389","https://openalex.org/W2103196307","https://openalex.org/W2112796928","https://openalex.org/W2131600418","https://openalex.org/W2132994929","https://openalex.org/W2145339207","https://openalex.org/W2145934551","https://openalex.org/W2146598871","https://openalex.org/W2147800946","https://openalex.org/W2149043954","https://openalex.org/W2163605009","https://openalex.org/W2787259794"],"related_works":["https://openalex.org/W6988599621","https://openalex.org/W93538108","https://openalex.org/W2607836663","https://openalex.org/W2167303412","https://openalex.org/W6996939854","https://openalex.org/W2112820504","https://openalex.org/W2010288268","https://openalex.org/W2740671916","https://openalex.org/W4327642284"],"abstract_inverted_index":{"When":[0],"reinforcement":[1],"learning":[2,49,83,99],"is":[3,31,66,156,204],"applied":[4],"to":[5,24,26,50,68,81,100,160,169,178,207,216,218],"large":[6],"state":[7],"spaces,":[8],"such":[9],"as":[10,45,140,142],"those":[11],"occurring":[12],"in":[13,62,172,223],"playing":[14],"board":[15],"games,":[16],"the":[17,28,59,84,102,115,161,170,182,188,193,208,213,221,234,244,249],"use":[18],"of":[19,95,134,184,192,248],"a":[20,131,151,199,254],"good":[21],"function":[22,30,46],"approximator":[23,47],"learn":[25,181,217],"approximate":[27],"value":[29,85],"very":[32],"important.":[33],"In":[34,87],"previous":[35],"research,":[36],"multi-layer":[37],"perceptrons":[38,97,129,149,197,252],"have":[39,150],"often":[40],"been":[41],"quite":[42],"successfully":[43],"used":[44],"for":[48,98],"play":[51,101],"particular":[52,76],"games":[53],"with":[54,130,239],"temporal":[55],"difference":[56],"learning.":[57],"With":[58],"recent":[60],"developments":[61],"deep":[63,235],"learning,":[64],"it":[65],"important":[67],"study":[69],"if":[70],"using":[71],"multiple":[72],"hidden":[73,135,138,153,201],"layers":[74,136],"or":[75],"network":[77,215,238],"structures":[78,94],"can":[79],"help":[80],"improve":[82],"function.":[86],"this":[88,264],"paper,":[89],"we":[90],"compare":[91,124],"five":[92],"different":[93,132],"multilayer":[96,128,148,196,251],"game":[103],"Tic-Tac-Toe":[104,173,230],"3D,":[105],"both":[106,257],"when":[107,112],"training":[108,113,262],"through":[109,258,261],"self-play":[110],"and":[111,164,260],"against":[114,253,263],"same":[116],"fixed":[117,255,265],"opponent":[118],"they":[119],"are":[120],"tested":[121],"against.":[122],"We":[123],"three":[125],"fully":[126,205],"connected":[127,159,206],"number":[133],"and/or":[137],"units,":[139],"well":[141],"two":[143,194],"structured":[144,147,195,236],"ones.":[145],"These":[146],"first":[152,209],"layer":[154,202],"that":[155,167,203,233],"only":[157],"sparsely":[158],"input":[162],"layer,":[163],"has":[165,198,243],"units":[166],"correspond":[168],"rows":[171],"3D.":[174],"This":[175],"allows":[176,212],"them":[177],"more":[179],"easily":[180],"contribution":[183],"specific":[185],"patterns":[186],"on":[187,229],"corresponding":[189],"rows.":[190],"One":[191],"second":[200],"one,":[210],"which":[211],"neural":[214,237],"non-linearly":[219],"integrate":[220],"information":[222],"these":[224],"detected":[225],"patterns.":[226],"The":[227],"results":[228],"3D":[231],"show":[232],"integrated":[240],"pattern":[241],"detectors":[242],"strongest":[245],"performance":[246],"out":[247],"compared":[250],"opponent,":[256],"self-training":[259],"opponent.":[266]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W2250031007","counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":2},{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":1},{"year":2018,"cited_by_count":2},{"year":2017,"cited_by_count":5},{"year":2016,"cited_by_count":2}],"updated_date":"2026-11-19T14:11:59.513116","created_date":"2016-06-24T00:00:00"}
