{"id":"https://openalex.org/W4407424147","doi":"https://doi.org/10.48550/arxiv.2502.07087","title":"Evaluating the Systematic Reasoning Abilities of Large Language Models through Graph Coloring","display_name":"Evaluating the Systematic Reasoning Abilities of Large Language Models through Graph Coloring","publication_year":2025,"publication_date":"2025-02-10","ids":{"openalex":"https://openalex.org/W4407424147","doi":"https://doi.org/10.48550/arxiv.2502.07087"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2502.07087","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.07087","pdf_url":"https://arxiv.org/pdf/2502.07087","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2502.07087","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5108168918","display_name":"Alex Heyman","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]},{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA","US"],"is_corresponding":false,"raw_author_name":"Heyman, Alex","raw_affiliation_strings":["Department of Electrical Engineering and Computer Science , York University , Toronto , ON , Canada","University of California , Los Angeles , CA , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering and Computer Science , York University , Toronto , ON , Canada","institution_ids":["https://openalex.org/I192455969"]},{"raw_affiliation_string":"University of California , Los Angeles , CA , USA","institution_ids":["https://openalex.org/I161318765"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5028769863","display_name":"Joel Zylberberg","orcid":"https://orcid.org/0000-0002-8208-5698","observed_orcids":["https://orcid.org/0000-0002-8208-5698"]},"institutions":[{"id":"https://openalex.org/I109736498","display_name":"Canadian Institute for Advanced Research","ror":"https://ror.org/01sdtdd95","country_code":"CA","type":"facility","lineage":["https://openalex.org/I109736498"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Zylberberg, Joel","raw_affiliation_strings":["Jules Stein Eye Institute ,","Learning in Machines and Brains Program , CIFAR , Toronto , ON , Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Jules Stein Eye Institute ,","institution_ids":[]},{"raw_affiliation_string":"Learning in Machines and Brains Program , CIFAR , Toronto , ON , Canada","institution_ids":["https://openalex.org/I109736498"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7651000022888184,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7651000022888184,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.05119999870657921,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13904","display_name":"Artificial Intelligence Applications","score":0.04919999837875366,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9369999766349792},{"id":"https://openalex.org/keywords/graph-coloring","display_name":"graph coloring","score":0.7960000038146973},{"id":"https://openalex.org/keywords/step-by-step-reasoning","display_name":"step-by-step reasoning","score":0.777999997138977},{"id":"https://openalex.org/keywords/problem-difficulty","display_name":"problem difficulty","score":0.4869999885559082},{"id":"https://openalex.org/keywords/llm-benchmarking","display_name":"LLM benchmarking","score":0.7440000176429749},{"id":"https://openalex.org/keywords/2-coloring","display_name":"two-coloring","score":0.6290000081062317},{"id":"https://openalex.org/keywords/claude-3-5-sonnet","display_name":"Claude 3.5 Sonnet","score":0.8080000281333923},{"id":"https://openalex.org/keywords/llama-3-1","display_name":"Llama 3.1","score":0.8519999980926514},{"id":"https://openalex.org/keywords/gemini-1-5-pro","display_name":"Gemini 1.5 Pro","score":0.9580000042915344},{"id":"https://openalex.org/keywords/gpt-4o","display_name":"GPT-4o","score":0.9739999771118164},{"id":"https://openalex.org/keywords/deepseek-r1","display_name":"DeepSeek-R1","score":0.765999972820282}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5998262763023376},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.5168131589889526},{"id":"https://openalex.org/C76946457","wikidata":"https://www.wikidata.org/wiki/Q504843","display_name":"Graph coloring","level":3,"score":0.41989022493362427},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3517611026763916},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.32781708240509033},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3150770366191864}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2502.07087","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.07087","pdf_url":"https://arxiv.org/pdf/2502.07087","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2502.07087","doi":"https://doi.org/10.48550/arxiv.2502.07087","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2502.07087","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2502.07087","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.07087","pdf_url":"https://arxiv.org/pdf/2502.07087","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4407424147.pdf","grobid_xml":"https://content.openalex.org/works/W4407424147.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4387838288","https://openalex.org/W4417098252","https://openalex.org/W4400376199","https://openalex.org/W4402502224","https://openalex.org/W4412889439","https://openalex.org/W4416036399","https://openalex.org/W4414813423","https://openalex.org/W7163344630","https://openalex.org/W7101696223","https://openalex.org/W7075997457"],"abstract_inverted_index":{"Contemporary":[0],"large":[1],"language":[2],"models":[3,107],"are":[4],"powerful":[5],"problem-solving":[6],"tools,":[7],"but":[8,99],"they":[9],"exhibit":[10,112],"weaknesses":[11],"in":[12,120,137,153,164],"their":[13],"reasoning":[14,37,156,183],"abilities":[15],"which":[16],"ongoing":[17],"research":[18],"seeks":[19],"to":[20,89,97,166],"mitigate.":[21],"We":[22,50,170],"investigate":[23],"graph":[24,175],"coloring":[25,176],"as":[26,42,44],"a":[27,66],"means":[28],"of":[29,46,68,141,160,181],"evaluating":[30],"an":[31],"LLM's":[32],"capacities":[33],"for":[34,124,128,191],"systematic":[35,155],"step-by-step":[36],"and":[38,63,77,110,126,130,157,178],"possibility":[39],"space":[40],"exploration,":[41],"well":[43],"effects":[45],"semantic":[47],"problem":[48,118],"framing.":[49],"test":[51],"Claude":[52],"3.5":[53],"Sonnet,":[54],"Llama":[55],"3.1":[56],"405B,":[57],"Gemini":[58],"1.5":[59],"Pro,":[60],"GPT-4o,":[61],"o1-mini,":[62],"DeepSeek-R1":[64],"on":[65,116],"dataset":[67],"$k$-coloring":[69],"problems":[70,92,184],"with":[71],"$2":[72],"\\leq":[73,75,81,83],"k":[74],"4$":[76],"vertex":[78],"count":[79],"$4":[80],"n":[82],"8$,":[84],"using":[85],"partial":[86],"algorithmic":[87],"solvers":[88],"further":[90,188],"categorize":[91],"by":[93],"difficulty.":[94],"In":[95],"addition":[96],"substantial":[98],"varying":[100],"framing":[101],"effects,":[102],"we":[103],"find":[104],"that":[105,172],"all":[106,121],"except":[108],"o1-mini":[109,125],"R1":[111],"$>60\\%$":[113],"error":[114],"rates":[115],"difficult":[117],"types":[119],"frames":[122],"($>15\\%$":[123],"$>10\\%$":[127],"R1),":[129],"no":[131],"model":[132],"achieves":[133],"perfect":[134],"accuracy":[135],"even":[136],"the":[138,149,158],"simple":[139],"domain":[140],"2-coloring":[142],"4-vertex":[143],"graphs.":[144],"Our":[145],"results":[146],"highlight":[147],"both":[148],"considerable":[150],"recent":[151],"progress":[152],"LLM":[154,192],"limits":[159],"its":[161],"reliability,":[162],"especially":[163],"relation":[165],"increasing":[167],"computational":[168],"costs.":[169],"expect":[171],"more":[173,185],"complex":[174],"problems,":[177],"procedural":[179],"generation":[180],"arbitrary-complexity":[182],"broadly,":[186],"offer":[187],"untapped":[189],"potential":[190],"benchmarking.":[193]},"counts_by_year":[],"updated_date":"2026-10-09T09:14:12.587421","created_date":"2025-02-13T00:00:00"}