{"id":"https://openalex.org/W7125585373","doi":"https://doi.org/10.1109/metroxraine66377.2025.11340083","title":"A Comparative Evaluation of Function-Calling LLMs in a Cognitive Architecture","display_name":"A Comparative Evaluation of Function-Calling LLMs in a Cognitive Architecture","publication_year":2025,"publication_date":"2025-10-22","ids":{"openalex":"https://openalex.org/W7125585373","doi":"https://doi.org/10.1109/metroxraine66377.2025.11340083"},"language":"en","primary_location":{"id":"doi:10.1109/metroxraine66377.2025.11340083","is_oa":false,"landing_page_url":"https://doi.org/10.1109/metroxraine66377.2025.11340083","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE International Conference on Metrology for eXtended Reality, Artificial Intelligence and Neural Engineering (MetroXRAINE)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5037237559","display_name":"M. Pardini","orcid":null},"institutions":[{"id":"https://openalex.org/I108290504","display_name":"University of Pisa","ror":"https://ror.org/03ad39j10","country_code":"IT","type":"education","lineage":["https://openalex.org/I108290504"]}],"countries":["IT"],"is_corresponding":true,"raw_author_name":"Marco Pardini","raw_affiliation_strings":["University of Pisa,Dept Information Engineering,Pisa,Italy"],"affiliations":[{"raw_affiliation_string":"University of Pisa,Dept Information Engineering,Pisa,Italy","institution_ids":["https://openalex.org/I108290504"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066840168","display_name":"Federico A. Galatolo","orcid":"https://orcid.org/0000-0001-7193-3754"},"institutions":[{"id":"https://openalex.org/I108290504","display_name":"University of Pisa","ror":"https://ror.org/03ad39j10","country_code":"IT","type":"education","lineage":["https://openalex.org/I108290504"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Federico A. Galatolo","raw_affiliation_strings":["University of Pisa,Dept Information Engineering,Pisa,Italy"],"affiliations":[{"raw_affiliation_string":"University of Pisa,Dept Information Engineering,Pisa,Italy","institution_ids":["https://openalex.org/I108290504"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016791461","display_name":"Lorenzo Cominelli","orcid":"https://orcid.org/0000-0001-8333-7538"},"institutions":[{"id":"https://openalex.org/I108290504","display_name":"University of Pisa","ror":"https://ror.org/03ad39j10","country_code":"IT","type":"education","lineage":["https://openalex.org/I108290504"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Lorenzo Cominelli","raw_affiliation_strings":["University of Pisa,Dept Information Engineering,Pisa,Italy"],"affiliations":[{"raw_affiliation_string":"University of Pisa,Dept Information Engineering,Pisa,Italy","institution_ids":["https://openalex.org/I108290504"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123756030","display_name":"Angelo De Marco","orcid":null},"institutions":[{"id":"https://openalex.org/I108290504","display_name":"University of Pisa","ror":"https://ror.org/03ad39j10","country_code":"IT","type":"education","lineage":["https://openalex.org/I108290504"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Angelo De Marco","raw_affiliation_strings":["University of Pisa,Dept Information Engineering,Pisa,Italy"],"affiliations":[{"raw_affiliation_string":"University of Pisa,Dept Information Engineering,Pisa,Italy","institution_ids":["https://openalex.org/I108290504"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123766077","display_name":"Mario G.C.A. Cimino","orcid":null},"institutions":[{"id":"https://openalex.org/I108290504","display_name":"University of Pisa","ror":"https://ror.org/03ad39j10","country_code":"IT","type":"education","lineage":["https://openalex.org/I108290504"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Mario G.C.A. Cimino","raw_affiliation_strings":["University of Pisa,Dept Information Engineering,Pisa,Italy"],"affiliations":[{"raw_affiliation_string":"University of Pisa,Dept Information Engineering,Pisa,Italy","institution_ids":["https://openalex.org/I108290504"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022849441","display_name":"Alberto Greco","orcid":"https://orcid.org/0000-0003-2387-9049"},"institutions":[{"id":"https://openalex.org/I108290504","display_name":"University of Pisa","ror":"https://ror.org/03ad39j10","country_code":"IT","type":"education","lineage":["https://openalex.org/I108290504"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Alberto Greco","raw_affiliation_strings":["University of Pisa,Dept Information Engineering,Pisa,Italy"],"affiliations":[{"raw_affiliation_string":"University of Pisa,Dept Information Engineering,Pisa,Italy","institution_ids":["https://openalex.org/I108290504"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5123732594","display_name":"Enzo Pasquale Scilingo","orcid":null},"institutions":[{"id":"https://openalex.org/I108290504","display_name":"University of Pisa","ror":"https://ror.org/03ad39j10","country_code":"IT","type":"education","lineage":["https://openalex.org/I108290504"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Enzo Pasquale Scilingo","raw_affiliation_strings":["University of Pisa,Dept Information Engineering,Pisa,Italy"],"affiliations":[{"raw_affiliation_string":"University of Pisa,Dept Information Engineering,Pisa,Italy","institution_ids":["https://openalex.org/I108290504"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5037237559"],"corresponding_institution_ids":["https://openalex.org/I108290504"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.79669021,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1078","last_page":"1083"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10709","display_name":"Social Robot Interaction and HRI","score":0.36230000853538513,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10709","display_name":"Social Robot Interaction and HRI","score":0.36230000853538513,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10465","display_name":"Neurobiology of Language and Bilingualism","score":0.12470000237226486,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.05689999833703041,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.6748999953269958},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5961999893188477},{"id":"https://openalex.org/keywords/cognition","display_name":"Cognition","score":0.5641000270843506},{"id":"https://openalex.org/keywords/preference","display_name":"Preference","score":0.5626999735832214},{"id":"https://openalex.org/keywords/cognitive-architecture","display_name":"Cognitive architecture","score":0.5300999879837036},{"id":"https://openalex.org/keywords/testbed","display_name":"Testbed","score":0.47029998898506165},{"id":"https://openalex.org/keywords/architecture","display_name":"Architecture","score":0.448199987411499},{"id":"https://openalex.org/keywords/through-the-lens-metering","display_name":"Through-the-lens metering","score":0.4169999957084656}],"concepts":[{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.6748999953269958},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5961999893188477},{"id":"https://openalex.org/C169900460","wikidata":"https://www.wikidata.org/wiki/Q2200417","display_name":"Cognition","level":2,"score":0.5641000270843506},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.5626999735832214},{"id":"https://openalex.org/C20854674","wikidata":"https://www.wikidata.org/wiki/Q4386060","display_name":"Cognitive architecture","level":3,"score":0.5300999879837036},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.5145999789237976},{"id":"https://openalex.org/C31395832","wikidata":"https://www.wikidata.org/wiki/Q1318674","display_name":"Testbed","level":2,"score":0.47029998898506165},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.448199987411499},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.4408000111579895},{"id":"https://openalex.org/C43091099","wikidata":"https://www.wikidata.org/wiki/Q1067788","display_name":"Through-the-lens metering","level":3,"score":0.4169999957084656},{"id":"https://openalex.org/C2776445246","wikidata":"https://www.wikidata.org/wiki/Q1792644","display_name":"Style (visual arts)","level":2,"score":0.4025000035762787},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.3804999887943268},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3801000118255615},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.3312000036239624},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.323199987411499},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.3222000002861023},{"id":"https://openalex.org/C150681269","wikidata":"https://www.wikidata.org/wiki/Q2380771","display_name":"Cognitive style","level":3,"score":0.3149000108242035},{"id":"https://openalex.org/C2776608160","wikidata":"https://www.wikidata.org/wiki/Q4785462","display_name":"Natural (archaeology)","level":2,"score":0.29739999771118164},{"id":"https://openalex.org/C2777601683","wikidata":"https://www.wikidata.org/wiki/Q6499736","display_name":"Vocabulary","level":2,"score":0.2851000130176544},{"id":"https://openalex.org/C89611455","wikidata":"https://www.wikidata.org/wiki/Q6804646","display_name":"Mechanism (biology)","level":2,"score":0.2768999934196472},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.2750999927520752},{"id":"https://openalex.org/C86658582","wikidata":"https://www.wikidata.org/wiki/Q1432778","display_name":"Social cognition","level":3,"score":0.27489998936653137},{"id":"https://openalex.org/C49630185","wikidata":"https://www.wikidata.org/wiki/Q6980675","display_name":"Natural experiment","level":2,"score":0.27070000767707825},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.2635999917984009}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/metroxraine66377.2025.11340083","is_oa":false,"landing_page_url":"https://doi.org/10.1109/metroxraine66377.2025.11340083","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE International Conference on Metrology for eXtended Reality, Artificial Intelligence and Neural Engineering (MetroXRAINE)","raw_type":"proceedings-article"},{"id":"pmh:oai:arpi.unipi.it:11568/1345528","is_oa":false,"landing_page_url":"https://hdl.handle.net/11568/1345528","pdf_url":null,"source":{"id":"https://openalex.org/S4377196265","display_name":"CINECA IRIS Institutial research information system (University of Pisa)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I108290504","host_organization_name":"University of Pisa","host_organization_lineage":["https://openalex.org/I108290504"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.5424764156341553,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":13,"referenced_works":["https://openalex.org/W1504607447","https://openalex.org/W1965662041","https://openalex.org/W1988545511","https://openalex.org/W2117553866","https://openalex.org/W2123341385","https://openalex.org/W4287147246","https://openalex.org/W4390722831","https://openalex.org/W4392811690","https://openalex.org/W4394828156","https://openalex.org/W4402241846","https://openalex.org/W4406892528","https://openalex.org/W4410560172","https://openalex.org/W4415796627"],"related_works":[],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"now":[4],"possess":[5],"function":[6,69,202],"calling":[7],"capabilities,":[8],"enabling":[9],"them":[10],"to":[11,32,209],"interact":[12],"with":[13,122,169],"external":[14],"tools":[15],"and":[16,54,64,85,93,149,183,193,211],"APIs.":[17],"Within":[18],"cognitive":[19],"architectures":[20],"for":[21,29],"social":[22,118],"robotics,":[23],"this":[24],"provides":[25],"a":[26,34,74,109,165,206],"robust":[27],"mechanism":[28],"an":[30,104],"LLM":[31],"orchestrate":[33],"set":[35],"of":[36,77,142,216],"discrete":[37],"functions-conceptually":[38],"echoing":[39],"the":[40,100,123,127,140,213],"brain's":[41],"functional":[42],"specificity-to":[43],"manage":[44],"operations":[45],"such":[46,103],"as":[47,99,180],"visual":[48],"perception,":[49],"auditory":[50],"processing,":[51],"speech":[52],"generation,":[53],"memory":[55],"access,":[56],"among":[57],"others.":[58],"However,":[59],"LLMs":[60,129,218],"exhibit":[61],"varying":[62],"propensities":[63],"strategies":[65,215],"when":[66],"utilizing":[67],"these":[68],"calls.":[70,196],"This":[71],"paper":[72],"presents":[73],"comparative":[75],"evaluation":[76],"four":[78],"LLMs,":[79],"differing":[80],"significantly":[81,160],"in":[82,102,116,139,188,219],"parameter":[83],"scale":[84],"origin:":[86],"Llama3":[87,157],"70b,":[88],"Gemma2":[89],"9b,":[90],"Mixtral":[91,186],"8x7b,":[92,187],"Phi3":[94],"mini":[95],"3.8b,":[96],"each":[97,120],"acting":[98],"orchestrator":[101],"architecture.":[105],"Our":[106],"testbed":[107],"involved":[108],"human-participant":[110],"study":[111],"(N=20)":[112],"where":[113],"individuals":[114],"engaged":[115],"ambiguous":[117],"scenarios,":[119],"interacting":[121],"architecture":[124],"driven":[125],"by":[126],"different":[128,217],"(4":[130],"trials":[131],"per":[132],"participant).":[133],"Results":[134],"revealed":[135],"statistically":[136],"significant":[137],"differences":[138],"frequency":[141],"Look":[143],"(F=13.62,":[144],"p<0.001),":[145,148],"Talk":[146],"(F=9.29,":[147],"Hear":[150],"(F=10.34,":[151],"p<0.001)":[152],"calls":[153,163],"across":[154],"LLMs.":[155],"Notably,":[156],"70b":[158],"made":[159],"more":[161,181],"\u2018Look\u2019":[162],"(M=3.45),":[164],"behavior":[166],"that":[167,200],"corresponded":[168],"strong":[170],"user":[171],"preference":[172],"(18/20),":[173],"suggesting":[174],"its":[175],"interaction":[176,214],"style":[177],"was":[178],"perceived":[179],"natural":[182],"contextually":[184],"aware.":[185],"contrast,":[189],"favored":[190],"\u2018Talk\u2019":[191],"(M=11.05)":[192],"\u2018Hear\u2019":[194],"(M=11.15)":[195],"These":[197],"findings":[198],"demonstrate":[199],"analyzing":[201],"call":[203],"patterns":[204],"offers":[205],"quantitative":[207],"lens":[208],"understand":[210],"compare":[212],"orchestrating":[220],"robotic":[221],"behavior.":[222]},"counts_by_year":[],"updated_date":"2026-01-25T23:08:32.236327","created_date":"2026-01-25T00:00:00"}