{"id":"https://openalex.org/W7164499543","doi":"https://doi.org/10.48550/arxiv.2606.13649","title":"Operadic consistency: a label-free signal for compositional reasoning failures in LLMs","display_name":"Operadic consistency: a label-free signal for compositional reasoning failures in LLMs","publication_year":2026,"publication_date":"2026-06-11","ids":{"openalex":"https://openalex.org/W7164499543","doi":"https://doi.org/10.48550/arxiv.2606.13649"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.13649","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.13649","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.13649","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5085323451","display_name":"Nathaniel Bottman","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bottman, Nathaniel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138545112","display_name":"Yinhong Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Yinhong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136502110","display_name":"Kyle Richardson","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Richardson, Kyle","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.5995000004768372,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.5995000004768372,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.15369999408721924,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.031300000846385956,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.612500011920929},{"id":"https://openalex.org/keywords/formalism","display_name":"Formalism (music)","score":0.5142999887466431},{"id":"https://openalex.org/keywords/iterated-function","display_name":"Iterated function","score":0.5005000233650208},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.448199987411499},{"id":"https://openalex.org/keywords/entropy","display_name":"Entropy (arrow of time)","score":0.3993000090122223},{"id":"https://openalex.org/keywords/signal","display_name":"SIGNAL (programming language)","score":0.3474000096321106},{"id":"https://openalex.org/keywords/piecewise","display_name":"Piecewise","score":0.32820001244544983}],"concepts":[{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.612500011920929},{"id":"https://openalex.org/C73301696","wikidata":"https://www.wikidata.org/wiki/Q5469984","display_name":"Formalism (music)","level":3,"score":0.5142999887466431},{"id":"https://openalex.org/C140479938","wikidata":"https://www.wikidata.org/wiki/Q5254619","display_name":"Iterated function","level":2,"score":0.5005000233650208},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4819999933242798},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.448199987411499},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4359000027179718},{"id":"https://openalex.org/C106301342","wikidata":"https://www.wikidata.org/wiki/Q4117933","display_name":"Entropy (arrow of time)","level":2,"score":0.3993000090122223},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3571000099182129},{"id":"https://openalex.org/C2779843651","wikidata":"https://www.wikidata.org/wiki/Q7390335","display_name":"SIGNAL (programming language)","level":2,"score":0.3474000096321106},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3391000032424927},{"id":"https://openalex.org/C164660894","wikidata":"https://www.wikidata.org/wiki/Q2037833","display_name":"Piecewise","level":2,"score":0.32820001244544983},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.305400013923645},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.2955000102519989},{"id":"https://openalex.org/C25810664","wikidata":"https://www.wikidata.org/wiki/Q44325","display_name":"Ontology","level":2,"score":0.29510000348091125},{"id":"https://openalex.org/C124681953","wikidata":"https://www.wikidata.org/wiki/Q339062","display_name":"Decomposition","level":2,"score":0.28600001335144043},{"id":"https://openalex.org/C2778571376","wikidata":"https://www.wikidata.org/wiki/Q1355821","display_name":"Frontier","level":2,"score":0.28200000524520874},{"id":"https://openalex.org/C33762810","wikidata":"https://www.wikidata.org/wiki/Q461671","display_name":"Data integrity","level":2,"score":0.2809000015258789},{"id":"https://openalex.org/C2776728590","wikidata":"https://www.wikidata.org/wiki/Q363948","display_name":"Contradiction","level":2,"score":0.275299996137619},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2687000036239624},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.2655999958515167},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.2540000081062317},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.25200000405311584}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.13649","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.13649","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.13649","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.13649","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","score":0.4098213315010071,"id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Detecting":[0],"LLM":[1],"reasoning":[2],"failures":[3],"at":[4,201,209],"inference":[5],"time":[6],"without":[7],"ground-truth":[8],"labels":[9],"has":[10],"motivated":[11],"a":[12,41,44,49,61,76,205],"wide":[13],"range":[14],"of":[15,64,218,225,251,277],"confidence":[16],"baselines,":[17],"including":[18],"self-consistency,":[19],"semantic":[20,166],"entropy,":[21],"and":[22,28,88,113,140,154,165,179,222],"P(True),":[23],"built":[24,36],"on":[25,90,101,138,152,168,233,262,275],"within-question":[26],"sampling":[27],"self-evaluation.":[29],"Operad":[30],"theory,":[31],"the":[32,55,65,115,157,176,180,210,242,247,253,278],"formalism":[33],"for":[34,175,187],"systems":[35],"by":[37,59],"iterated":[38],"substitution,":[39],"suggests":[40],"complementary":[42],"diagnostic:":[43],"model's":[45,248],"direct":[46],"answer":[47,56],"to":[48,84,148,184,220,227],"compositional":[50],"query":[51],"should":[52],"agree":[53],"with":[54,99,120,270],"it":[57],"produces":[58],"composing":[60],"stated":[62],"decomposition":[63,243],"same":[66,195,254],"query.":[67],"We":[68],"instantiate":[69],"this":[70],"idea":[71],"as":[72],"operadic":[73],"consistency":[74],"(OC),":[75],"per-question":[77,158],"signal.":[78],"Across":[79],"twelve":[80],"instruction-tuned":[81],"LLMs":[82],"(4B":[83],"671B":[85],"parameters,":[86],"open-weights":[87],"closed-source)":[89],"four":[91,127],"multi-hop":[92],"QA":[93],"datasets,":[94],"OC":[95,137,160,177],"is":[96,114,182,244],"strongly":[97],"correlated":[98],"accuracy":[100],"every":[102,169,234],"dataset":[103,170],"(Pearson":[104],"$r":[105,121,149],"\\in":[106],"[0.86,":[107],"0.94]$,":[108],"all":[109,126,263],"$p":[110,172],"\\leq":[111,173,192],"0.0004$),":[112],"only":[116],"signal":[117,196],"we":[118],"evaluate":[119],"\\geq":[122],"0.85$":[123],"uniformly":[124],"across":[125],"datasets.":[128],"Chain-of-thought":[129],"self-consistency":[130],"(CoT-SC;":[131],"Wang":[132],"et":[133],"al.,":[134],"2023)":[135],"matches":[136],"HotpotQA":[139],"DROP":[141],"($r":[142],"=":[143,213],"0.93,":[144],"0.87$)":[145],"but":[146],"drops":[147],"\\approx":[150],"0.45$":[151],"MuSiQue":[153],"StrategyQA.":[155],"At":[156],"level,":[159],"contributes":[161],"information":[162],"beyond":[163],"CoT-SC":[164,207],"entropy":[167],"(cluster-robust":[171],"10^{-16}$":[174],"coefficient),":[178],"conclusion":[181],"robust":[183],"additionally":[185],"controlling":[186],"constructed":[188],"decomposition-aware":[189],"baselines":[190],"($p":[191],"10^{-13}$).":[193],"The":[194],"yields":[197],"selective-prediction":[198,259],"improvements":[199],"(accuracy":[200],"fixed":[202],"coverage)":[203],"over":[204],"tuned":[206],"baseline":[208],"equal-cost":[211,255],"$K":[212],"3$":[214],"budget":[215],"(AUARC":[216],"lifts":[217,224],"+0.086":[219],"+0.096":[221],"AUROC":[223],"+0.092":[226],"+0.164;":[228],"95%":[229,271],"CIs":[230,272],"exclude":[231],"zero":[232,274],"cell).":[235],"On":[236],"five":[237],"frontier":[238],"thinking":[239],"models,":[240],"where":[241],"extracted":[245],"from":[246],"own":[249],"chain":[250],"thought,":[252],"comparison":[256],"gives":[257],"positive":[258],"point-estimate":[260],"lift":[261],"16":[264],"(dataset,":[265],"budget,":[266],"metric)":[267],"cells":[268],"tested,":[269],"excluding":[273],"12":[276],"16.":[279]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-13T00:00:00"}