{"id":"https://openalex.org/W7204850475","doi":"https://doi.org/10.48550/arxiv.2608.27750","title":"The Calls are Coming from Inside the Model: Investigating Probe-based Detection of Tool-Calling Errors in LLMs","display_name":"The Calls are Coming from Inside the Model: Investigating Probe-based Detection of Tool-Calling Errors in LLMs","publication_year":2026,"publication_date":"2026-08-27","ids":{"openalex":"https://openalex.org/W7204850475","doi":"https://doi.org/10.48550/arxiv.2608.27750"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2608.27750","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.27750","pdf_url":"https://arxiv.org/pdf/2608.27750","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2608.27750","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5090824296","display_name":"Eric Yeats","orcid":"https://orcid.org/0000-0002-5470-3380","observed_orcids":["https://orcid.org/0000-0002-5470-3380"]},"institutions":[{"id":"https://openalex.org/I130440993","display_name":"National Security Agency","ror":"https://ror.org/0047bvr32","country_code":"US","type":"government","lineage":["https://openalex.org/I130440993","https://openalex.org/I1330347796"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yeats, Eric","raw_affiliation_strings":["National Security Agency"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Security Agency","institution_ids":["https://openalex.org/I130440993"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008962142","display_name":"Brendan Kennedy","orcid":"https://orcid.org/0000-0001-7252-7475","observed_orcids":["https://orcid.org/0000-0001-7252-7475"]},"institutions":[{"id":"https://openalex.org/I130440993","display_name":"National Security Agency","ror":"https://ror.org/0047bvr32","country_code":"US","type":"government","lineage":["https://openalex.org/I130440993","https://openalex.org/I1330347796"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kennedy, Brendan","raw_affiliation_strings":["National Security Agency"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Security Agency","institution_ids":["https://openalex.org/I130440993"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112588337","display_name":"Loc Truong","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I142606810","display_name":"Pacific Northwest National Laboratory","ror":"https://ror.org/05h992307","country_code":"US","type":"facility","lineage":["https://openalex.org/I1325736334","https://openalex.org/I1330989302","https://openalex.org/I142606810","https://openalex.org/I39565521"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Truong, Loc","raw_affiliation_strings":["Pacific Northwest National Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Pacific Northwest National Laboratory","institution_ids":["https://openalex.org/I142606810"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065211007","display_name":"John Buckheit","orcid":"https://orcid.org/0000-0003-4499-1718","observed_orcids":["https://orcid.org/0000-0003-4499-1718"]},"institutions":[{"id":"https://openalex.org/I142606810","display_name":"Pacific Northwest National Laboratory","ror":"https://ror.org/05h992307","country_code":"US","type":"facility","lineage":["https://openalex.org/I1325736334","https://openalex.org/I1330989302","https://openalex.org/I142606810","https://openalex.org/I39565521"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Buckheit, John","raw_affiliation_strings":["Pacific Northwest National Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Pacific Northwest National Laboratory","institution_ids":["https://openalex.org/I142606810"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077314223","display_name":"Jung H. Lee","orcid":"https://orcid.org/0000-0001-8568-8163","observed_orcids":["https://orcid.org/0000-0001-8568-8163"]},"institutions":[{"id":"https://openalex.org/I142606810","display_name":"Pacific Northwest National Laboratory","ror":"https://ror.org/05h992307","country_code":"US","type":"facility","lineage":["https://openalex.org/I1325736334","https://openalex.org/I1330989302","https://openalex.org/I142606810","https://openalex.org/I39565521"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lee, Jung","raw_affiliation_strings":["Pacific Northwest National Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Pacific Northwest National Laboratory","institution_ids":["https://openalex.org/I142606810"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077651154","display_name":"Jesse Robert Friedbaum","orcid":"https://orcid.org/0009-0009-4351-6369","observed_orcids":["https://orcid.org/0009-0009-4351-6369"]},"institutions":[{"id":"https://openalex.org/I142606810","display_name":"Pacific Northwest National Laboratory","ror":"https://ror.org/05h992307","country_code":"US","type":"facility","lineage":["https://openalex.org/I1325736334","https://openalex.org/I1330989302","https://openalex.org/I142606810","https://openalex.org/I39565521"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Friedbaum, Jesse","raw_affiliation_strings":["Pacific Northwest National Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Pacific Northwest National Laboratory","institution_ids":["https://openalex.org/I142606810"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071202341","display_name":"John A. Emanuello","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I130440993","display_name":"National Security Agency","ror":"https://ror.org/0047bvr32","country_code":"US","type":"government","lineage":["https://openalex.org/I130440993","https://openalex.org/I1330347796"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Emanuello, John","raw_affiliation_strings":["National Security Agency"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Security Agency","institution_ids":["https://openalex.org/I130440993"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5066361433","display_name":"Henry Kvinge","orcid":"https://orcid.org/0000-0003-4108-1364","observed_orcids":["https://orcid.org/0000-0003-4108-1364"]},"institutions":[{"id":"https://openalex.org/I142606810","display_name":"Pacific Northwest National Laboratory","ror":"https://ror.org/05h992307","country_code":"US","type":"facility","lineage":["https://openalex.org/I1325736334","https://openalex.org/I1330989302","https://openalex.org/I142606810","https://openalex.org/I39565521"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kvinge, Henry","raw_affiliation_strings":["Pacific Northwest National Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Pacific Northwest National Laboratory","institution_ids":["https://openalex.org/I142606810"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.5181000232696533,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.5181000232696533,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.19370000064373016,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.11699999868869781,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/linear-array-transducer","display_name":"linear array transducer","score":0.6129999756813049},{"id":"https://openalex.org/keywords/hidden-states","display_name":"hidden states","score":0.7739999890327454},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9279999732971191},{"id":"https://openalex.org/keywords/function-calling","display_name":"function calling","score":0.7630000114440918},{"id":"https://openalex.org/keywords/model-size","display_name":"model size","score":0.41499999165534973},{"id":"https://openalex.org/keywords/post-training","display_name":"post-training","score":0.6420000195503235}],"concepts":[{"id":"https://openalex.org/C98184364","wikidata":"https://www.wikidata.org/wiki/Q1780131","display_name":"Argument (complex analysis)","level":2,"score":0.652400016784668},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5324000120162964},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.5299000144004822},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.44600000977516174},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.42089998722076416},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3693999946117401},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3188000023365021},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.3156000077724457},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.31380000710487366},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.2955999970436096},{"id":"https://openalex.org/C113843644","wikidata":"https://www.wikidata.org/wiki/Q901882","display_name":"Interface (matter)","level":4,"score":0.27950000762939453},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.27630001306533813},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2694999873638153},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.26840001344680786},{"id":"https://openalex.org/C3019719930","wikidata":"https://www.wikidata.org/wiki/Q3910099","display_name":"Predictive value","level":2,"score":0.25839999318122864},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.25270000100135803},{"id":"https://openalex.org/C125620115","wikidata":"https://www.wikidata.org/wiki/Q845249","display_name":"Logging","level":2,"score":0.2524999976158142}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2608.27750","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.27750","pdf_url":"https://arxiv.org/pdf/2608.27750","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.27750","doi":"https://doi.org/10.48550/arxiv.2608.27750","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.27750","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2608.27750","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.27750","pdf_url":"https://arxiv.org/pdf/2608.27750","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7204850475.pdf","grobid_xml":"https://content.openalex.org/works/W7204850475.grobid-xml"},"referenced_works_count":10,"referenced_works":["https://openalex.org/W4285429195","https://openalex.org/W4310926773","https://openalex.org/W4322718191","https://openalex.org/W4327810158","https://openalex.org/W4383987670","https://openalex.org/W4388514540","https://openalex.org/W4391631744","https://openalex.org/W4407185554","https://openalex.org/W7120123770","https://openalex.org/W7128628274"],"related_works":["https://openalex.org/W7172433968","https://openalex.org/W7172513554","https://openalex.org/W4404649838","https://openalex.org/W7120123770","https://openalex.org/W7167627487","https://openalex.org/W4404792967","https://openalex.org/W7160906306","https://openalex.org/W7171959633","https://openalex.org/W4415961327","https://openalex.org/W7204251673"],"abstract_inverted_index":{"The":[0],"hidden":[1],"states":[2],"of":[3,28,44,52,61,96,145,150],"large":[4],"language":[5],"models":[6],"(LLMs)":[7],"are":[8,143],"known":[9],"to":[10,15,24,65,92,147],"capture":[11],"rich":[12],"information":[13],"relating":[14],"model":[16,130,135],"knowledge":[17],"and":[18,30,134],"behavior":[19],"that":[20,86,107,141],"can":[21],"be":[22,119],"hard":[23],"extract":[25],"from":[26,103],"examination":[27],"input":[29],"output":[31],"alone.":[32],"As":[33],"LLM-based":[34],"systems":[35],"increasingly":[36],"interface":[37],"with":[38],"the":[39,59,78,109,113],"external":[40],"world,":[41],"one":[42],"area":[43],"concern":[45],"is":[46,88,153],"detecting":[47],"incorrect":[48,67],"or":[49],"improper":[50],"use":[51],"tools.":[53],"Motivated":[54],"by":[55,121],"this,":[56],"we":[57,84],"study":[58],"effectiveness":[60],"using":[62,104],"linear":[63],"probes":[64,142],"detect":[66],"tool-calls,":[68],"measuring":[69],"probe":[70],"efficacy":[71],"across":[72],"18":[73],"tool-calling":[74,98],"LLMs":[75],"evaluated":[76],"on":[77],"Berkeley":[79],"Function":[80],"Calling":[81],"Leaderboard.":[82],"Overall,":[83],"find":[85],"probing":[87,132],"an":[89,105],"effective":[90],"means":[91],"catch":[93],"a":[94],"range":[95],"different":[97],"errors,":[99,151],"including":[100],"errors":[101],"arising":[102],"argument":[106],"has":[108],"wrong":[110],"value":[111],"but":[112],"correct":[114],"type,":[115],"which":[116,152],"might":[117],"not":[118],"recorded":[120],"standard":[122],"logging":[123],"frameworks.":[124],"Important":[125],"factors":[126],"in":[127,155],"success":[128],"include":[129],"size,":[131],"layer,":[133],"post-training":[136],"type.":[137],"We":[138],"also":[139],"show":[140],"capable":[144],"generalizing":[146],"novel":[148],"types":[149],"critical":[154],"real":[156],"world":[157],"deployments.":[158]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-11-12T12:28:57.059296","created_date":"2026-09-01T00:00:00"}