{"id":"https://openalex.org/W4283793451","doi":"https://doi.org/10.1609/aaai.v36i10.21390","title":"Chess as a Testbed for Language Model State Tracking","display_name":"Chess as a Testbed for Language Model State Tracking","publication_year":2022,"publication_date":"2022-06-28","ids":{"openalex":"https://openalex.org/W4283793451","doi":"https://doi.org/10.1609/aaai.v36i10.21390","mag":"3133800575"},"language":"en","primary_location":{"id":"doi:10.1609/aaai.v36i10.21390","doi":"https://doi.org/10.1609/aaai.v36i10.21390","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v36i10.21390","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://doi.org/10.1609/aaai.v36i10.21390","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5073890145","display_name":"Shubham Toshniwal","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I160992636","display_name":"Toyota Technological Institute at Chicago","ror":"https://ror.org/02sn5gb64","country_code":"US","type":"education","lineage":["https://openalex.org/I160992636"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shubham Toshniwal","raw_affiliation_strings":["TTI Chicago"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"TTI Chicago","institution_ids":["https://openalex.org/I160992636"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073227709","display_name":"Sam Wiseman","orcid":"https://orcid.org/0000-0003-0923-1086","observed_orcids":["https://orcid.org/0000-0003-0923-1086"]},"institutions":[{"id":"https://openalex.org/I170897317","display_name":"Duke University","ror":"https://ror.org/00py81415","country_code":"US","type":"education","lineage":["https://openalex.org/I170897317"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sam Wiseman","raw_affiliation_strings":["Duke University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Duke University","institution_ids":["https://openalex.org/I170897317"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015602781","display_name":"Karen Livescu","orcid":"https://orcid.org/0000-0003-4962-946X","observed_orcids":["https://orcid.org/0000-0003-4962-946X"]},"institutions":[{"id":"https://openalex.org/I160992636","display_name":"Toyota Technological Institute at Chicago","ror":"https://ror.org/02sn5gb64","country_code":"US","type":"education","lineage":["https://openalex.org/I160992636"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Karen Livescu","raw_affiliation_strings":["TTI Chicago"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"TTI Chicago","institution_ids":["https://openalex.org/I160992636"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5081022650","display_name":"Kevin Gimpel","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I160992636","display_name":"Toyota Technological Institute at Chicago","ror":"https://ror.org/02sn5gb64","country_code":"US","type":"education","lineage":["https://openalex.org/I160992636"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kevin Gimpel","raw_affiliation_strings":["TTI Chicago"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"TTI Chicago","institution_ids":["https://openalex.org/I160992636"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.2477,"has_fulltext":true,"cited_by_count":42,"citation_normalized_percentile":{"value":0.94144261,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":87,"max":99},"biblio":{"volume":"36","issue":"10","first_page":"11385","last_page":"11393"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.586899995803833,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.586899995803833,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11574","display_name":"Artificial Intelligence in Games","score":0.34200000762939453,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.04639999940991402,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/chess","display_name":"chess","score":0.7730000019073486},{"id":"https://openalex.org/keywords/chess-notation","display_name":"chess notation","score":0.7059999704360962},{"id":"https://openalex.org/keywords/full-attention","display_name":"full attention","score":0.6660000085830688},{"id":"https://openalex.org/keywords/history-of-games","display_name":"history of games","score":0.5440000295639038},{"id":"https://openalex.org/keywords/transformer-based-language-models","display_name":"transformer language models","score":0.7070000171661377},{"id":"https://openalex.org/keywords/move-sequences","display_name":"move sequences","score":0.5720000267028809}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7724384665489197},{"id":"https://openalex.org/C31395832","wikidata":"https://www.wikidata.org/wiki/Q1318674","display_name":"Testbed","level":2,"score":0.7595502138137817},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.6908013224601746},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.6216021180152893},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5820537805557251},{"id":"https://openalex.org/C45357846","wikidata":"https://www.wikidata.org/wiki/Q2001982","display_name":"Notation","level":2,"score":0.5279741883277893},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5027084350585938},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.43068578839302063},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4130479693412781},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.33445632457733154},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.11877146363258362},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.08780255913734436},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.1609/aaai.v36i10.21390","doi":"https://doi.org/10.1609/aaai.v36i10.21390","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v36i10.21390","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2102.13249","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2102.13249","pdf_url":"https://arxiv.org/pdf/2102.13249","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:ojs.aaai.org:article/21390","doi":null,"is_oa":true,"landing_page_url":"https://ojs.aaai.org/index.php/AAAI/article/view/21390","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/21390/21139","source":{"id":"https://openalex.org/S4306400155","display_name":"AAAI Publications (The Association for the Advancement of Artificial Intelligence (AAAI))","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801652627","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/I2801652627"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2159-5399","raw_type":"info:eu-repo/semantics/article"},{"id":"mag:3133800575","doi":null,"is_oa":true,"landing_page_url":"http://export.arxiv.org/pdf/2102.13249","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.2102.13249","doi":"https://doi.org/10.48550/arxiv.2102.13249","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2102.13249","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1609/aaai.v36i10.21390","doi":"https://doi.org/10.1609/aaai.v36i10.21390","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v36i10.21390","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4283793451.pdf","grobid_xml":"https://content.openalex.org/works/W4283793451.grobid-xml"},"referenced_works_count":51,"referenced_works":["https://openalex.org/W1522301498","https://openalex.org/W1525961042","https://openalex.org/W1527575280","https://openalex.org/W1532325895","https://openalex.org/W1544827683","https://openalex.org/W2064675550","https://openalex.org/W2112184938","https://openalex.org/W2126209950","https://openalex.org/W2466175319","https://openalex.org/W2473344385","https://openalex.org/W2509966261","https://openalex.org/W2515741950","https://openalex.org/W2516090925","https://openalex.org/W2529194139","https://openalex.org/W2549835527","https://openalex.org/W2606377603","https://openalex.org/W2626778328","https://openalex.org/W2627585944","https://openalex.org/W2692059227","https://openalex.org/W2763421725","https://openalex.org/W2766569996","https://openalex.org/W2810346659","https://openalex.org/W2890208753","https://openalex.org/W2896457183","https://openalex.org/W2902907165","https://openalex.org/W2908854766","https://openalex.org/W2948176064","https://openalex.org/W2962776659","https://openalex.org/W2963341956","https://openalex.org/W2964121744","https://openalex.org/W2965647350","https://openalex.org/W2967853777","https://openalex.org/W2970476646","https://openalex.org/W2970862333","https://openalex.org/W2970971581","https://openalex.org/W2971869958","https://openalex.org/W2980282514","https://openalex.org/W2994673210","https://openalex.org/W3001279689","https://openalex.org/W3015449890","https://openalex.org/W3030163527","https://openalex.org/W3034573343","https://openalex.org/W3034723486","https://openalex.org/W3035597164","https://openalex.org/W3047988254","https://openalex.org/W3091156754","https://openalex.org/W3125056032","https://openalex.org/W4213009331","https://openalex.org/W6631834165","https://openalex.org/W6739518724","https://openalex.org/W6739847781"],"related_works":["https://openalex.org/W3133800575","https://openalex.org/W3127745465","https://openalex.org/W3211354215","https://openalex.org/W7162811713","https://openalex.org/W7127549097","https://openalex.org/W7161719521","https://openalex.org/W3047988254","https://openalex.org/W4296363276","https://openalex.org/W7078362986","https://openalex.org/W4400701072"],"abstract_inverted_index":{"Transformer":[0],"language":[1,9,17,44,95,137,178],"models":[2,26,96,138],"have":[3],"made":[4],"tremendous":[5],"strides":[6],"in":[7,156],"natural":[8,16,52],"understanding":[10],"tasks.":[11],"However,":[12],"the":[13,29,33,41,47,67,77,144,172],"complexity":[14],"of":[15,43,49,70,135,176],"makes":[18],"it":[19],"challenging":[20],"to":[21,99,122,143],"ascertain":[22],"how":[23],"accurately":[24],"these":[25],"are":[27],"tracking":[28],"world":[30,78],"state":[31,124],"underlying":[32],"text.":[34],"Motivated":[35],"by":[36],"this":[37,152,163],"issue,":[38],"we":[39,64],"consider":[40],"task":[42],"modeling":[45],"for":[46,74,168],"game":[48,146],"chess.":[50],"Unlike":[51],"language,":[53],"chess":[54,71],"notations":[55],"describe":[56],"a":[57,157,166],"simple,":[58],"constrained,":[59],"and":[60,102,174],"deterministic":[61],"domain.":[62],"Moreover,":[63],"observe":[65],"that":[66],"appropriate":[68],"choice":[69],"notation":[72],"allows":[73],"directly":[75],"probing":[76],"state,":[79],"without":[80],"requiring":[81],"any":[82],"additional":[83],"probing-related":[84],"machinery.":[85],"We":[86,161],"find":[87],"that:":[88],"(a)":[89],"With":[90],"enough":[91],"training":[92,118,127],"data,":[93],"transformer":[94,136,177],"can":[97,128],"learn":[98],"track":[100],"pieces":[101],"predict":[103],"legal":[104],"moves":[105],"with":[106],"high":[107],"accuracy":[108],"when":[109],"trained":[110],"solely":[111],"on":[112,141,171],"move":[113],"sequences.":[114],"(b)":[115],"For":[116],"small":[117],"sets":[119],"providing":[120],"access":[121,142],"board":[123],"information":[125],"during":[126],"yield":[129],"significant":[130,158],"improvements.":[131],"(c)":[132],"The":[133],"success":[134],"is":[139],"dependent":[140],"entire":[145],"history":[147],"i.e.":[148],"\u201cfull":[149],"attention\u201d.":[150],"Approximating":[151],"full":[153],"attention":[154],"results":[155],"performance":[159],"drop.":[160],"propose":[162],"testbed":[164],"as":[165],"benchmark":[167],"future":[169],"work":[170],"development":[173],"analysis":[175],"models.":[179]},"counts_by_year":[{"year":2026,"cited_by_count":10},{"year":2025,"cited_by_count":10},{"year":2024,"cited_by_count":8},{"year":2023,"cited_by_count":7},{"year":2022,"cited_by_count":6},{"year":2021,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2022-07-05T00:00:00"}