{"id":"https://openalex.org/W7134823312","doi":"https://doi.org/10.48550/arxiv.2603.06636","title":"SmartBench: Evaluating LLMs in Smart Homes with Anomalous Device States and Behavioral Contexts","display_name":"SmartBench: Evaluating LLMs in Smart Homes with Anomalous Device States and Behavioral Contexts","publication_year":2026,"publication_date":"2026-02-24","ids":{"openalex":"https://openalex.org/W7134823312","doi":"https://doi.org/10.48550/arxiv.2603.06636"},"language":null,"primary_location":{"id":"pmh:doi:10.48550/arxiv.2603.06636","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"type":"article","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":null,"any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5128675425","display_name":"Qingsong Zou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zou, Qingsong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008360393","display_name":"Zhi Yan","orcid":"https://orcid.org/0000-0001-8251-9786"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yan, Zhi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128644851","display_name":"Zhiyao Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Zhiyao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038862367","display_name":"Kuofeng Gao","orcid":"https://orcid.org/0000-0002-5667-8238"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gao, Kuofeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033074326","display_name":"Jingyu Xiao","orcid":"https://orcid.org/0000-0002-2394-2995"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiao, Jingyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5128688050","display_name":"Yong Jiang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jiang, Yong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.28912189,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.5087000131607056,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.5087000131607056,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.07999999821186066,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10444","display_name":"Context-Aware Activity Recognition Systems","score":0.0608999989926815,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/home-automation","display_name":"Home automation","score":0.6342999935150146},{"id":"https://openalex.org/keywords/anomaly-detection","display_name":"Anomaly detection","score":0.5860000252723694},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.5284000039100647},{"id":"https://openalex.org/keywords/anomaly","display_name":"Anomaly (physics)","score":0.5267999768257141},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5217000246047974},{"id":"https://openalex.org/keywords/mainstream","display_name":"Mainstream","score":0.5142999887466431},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.5042999982833862}],"concepts":[{"id":"https://openalex.org/C507571656","wikidata":"https://www.wikidata.org/wiki/Q848436","display_name":"Home automation","level":2,"score":0.6342999935150146},{"id":"https://openalex.org/C739882","wikidata":"https://www.wikidata.org/wiki/Q3560506","display_name":"Anomaly detection","level":2,"score":0.5860000252723694},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.5284000039100647},{"id":"https://openalex.org/C12997251","wikidata":"https://www.wikidata.org/wiki/Q567560","display_name":"Anomaly (physics)","level":2,"score":0.5267999768257141},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5217000246047974},{"id":"https://openalex.org/C2777617010","wikidata":"https://www.wikidata.org/wiki/Q18957","display_name":"Mainstream","level":2,"score":0.5142999887466431},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.5042999982833862},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4893999993801117},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.44209998846054077},{"id":"https://openalex.org/C108827166","wikidata":"https://www.wikidata.org/wiki/Q175975","display_name":"Internet privacy","level":1,"score":0.38850000500679016},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.36000001430511475},{"id":"https://openalex.org/C172195944","wikidata":"https://www.wikidata.org/wiki/Q541265","display_name":"Ubiquitous computing","level":2,"score":0.3596000075340271},{"id":"https://openalex.org/C152223200","wikidata":"https://www.wikidata.org/wiki/Q3055471","display_name":"Smart environment","level":3,"score":0.33899998664855957},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.33799999952316284},{"id":"https://openalex.org/C2777103469","wikidata":"https://www.wikidata.org/wiki/Q1231558","display_name":"Smart city","level":3,"score":0.3140000104904175},{"id":"https://openalex.org/C194232998","wikidata":"https://www.wikidata.org/wiki/Q1606712","display_name":"Transition (genetics)","level":3,"score":0.2736000120639801},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.272599995136261},{"id":"https://openalex.org/C201638289","wikidata":"https://www.wikidata.org/wiki/Q457396","display_name":"Ambient intelligence","level":2,"score":0.2547000050544739}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:doi:10.48550/arxiv.2603.06636","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"doi:10.48550/arxiv.2603.06636","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.06636","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:doi:10.48550/arxiv.2603.06636","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Due":[0],"to":[1,23,37,70,211],"the":[2,68,73,86,108,123,219],"strong":[3],"context-awareness":[4],"capabilities":[5,111],"demonstrated":[6],"by":[7],"large":[8],"language":[9],"models":[10,163],"(LLMs),":[11],"recent":[12],"research":[13],"has":[14],"begun":[15],"exploring":[16],"their":[17,29],"integration":[18],"into":[19],"smart":[20,64,115,125,202,220],"home":[21,65,74,116,126,203,221],"assistants":[22,66,204],"help":[24],"users":[25],"manage":[26],"and":[27,42,53,97,134,142,182,214],"adjust":[28],"living":[30],"environments.":[31],"While":[32],"LLMs":[33,152],"have":[34],"been":[35],"shown":[36],"effectively":[38,212],"understand":[39],"user":[40,55],"needs":[41],"provide":[43,98],"appropriate":[44],"responses,":[45],"most":[46,161],"existing":[47],"studies":[48],"primarily":[49],"focus":[50],"on":[51,153,178,186],"interpreting":[52],"executing":[54],"behaviors":[56],"or":[57,103],"instructions.":[58],"However,":[59],"a":[60,100],"critical":[61],"function":[62],"of":[63,112,192],"is":[67,76,95,122,225],"ability":[69],"detect":[71,213],"when":[72],"environment":[75],"in":[77,218],"an":[78,92,190],"anomalous":[79,93,135,143,216],"state.":[80],"This":[81],"involves":[82],"two":[83],"key":[84],"requirements:":[85],"LLM":[87],"must":[88],"accurately":[89],"determine":[90],"whether":[91],"condition":[94],"present,":[96],"either":[99],"clear":[101],"explanation":[102],"actionable":[104],"suggestions.":[105],"To":[106],"enhance":[107],"anomaly":[109,167,180],"detection":[110,168,176],"next-generation":[113,200],"LLM-based":[114,201],"assistants,":[117],"we":[118],"introduce":[119],"SmartBench,":[120],"which":[121],"first":[124],"dataset":[127,224],"designed":[128],"for":[129],"LLMs,":[130],"containing":[131],"both":[132],"normal":[133,141],"device":[136,144],"states":[137],"as":[138,140],"well":[139],"state":[145],"transition":[146],"contexts.":[147],"We":[148],"evaluate":[149],"13":[150],"mainstream":[151],"this":[154],"benchmark.":[155],"The":[156],"experimental":[157,196],"results":[158,197],"show":[159],"that":[160,199],"state-of-the-art":[162],"cannot":[164],"achieve":[165],"good":[166],"performance.":[169],"For":[170],"example,":[171],"Claude-Sonnet-4.5":[172],"achieves":[173],"only":[174,193],"66.1%":[175],"accuracy":[177,191],"context-independent":[179],"categories,":[181],"performs":[183],"even":[184],"worse":[185],"context-dependent":[187],"anomalies,":[188],"with":[189],"57.8%.":[194],"More":[195],"suggest":[198],"are":[205],"still":[206],"far":[207],"from":[208],"being":[209],"able":[210],"handle":[215],"conditions":[217],"environment.":[222],"Our":[223],"publicly":[226],"available":[227],"at":[228],"https://github.com/horizonsinzqs/SmartBench.":[229]},"counts_by_year":[],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2026-03-11T00:00:00"}