{"id":"https://openalex.org/W7155620697","doi":"https://doi.org/10.1145/3785022.3785109","title":"A Novel Approach to Evaluating the Effectiveness of Large Language Models for Multimodal Analysis of Embodied Learning in Classrooms","display_name":"A Novel Approach to Evaluating the Effectiveness of Large Language Models for Multimodal Analysis of Embodied Learning in Classrooms","publication_year":2026,"publication_date":"2026-04-25","ids":{"openalex":"https://openalex.org/W7155620697","doi":"https://doi.org/10.1145/3785022.3785109"},"language":null,"primary_location":{"id":"doi:10.1145/3785022.3785109","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3785022.3785109","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the LAK26: 16th International Learning Analytics and Knowledge Conference","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3785022.3785109","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5076518309","display_name":"Joyce Horn Fonteles","orcid":"https://orcid.org/0000-0001-9862-8960"},"institutions":[{"id":"https://openalex.org/I200719446","display_name":"Vanderbilt University","ror":"https://ror.org/02vm5rt34","country_code":"US","type":"education","lineage":["https://openalex.org/I200719446"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Joyce Horn Fonteles","raw_affiliation_strings":["Vanderbilt University, Nashville, USA"],"raw_orcid":"https://orcid.org/0000-0001-9862-8960","affiliations":[{"raw_affiliation_string":"Vanderbilt University, Nashville, USA","institution_ids":["https://openalex.org/I200719446"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119949207","display_name":"Nithin Sivakumaran","orcid":null},"institutions":[{"id":"https://openalex.org/I114027177","display_name":"University of North Carolina at Chapel Hill","ror":"https://ror.org/0130frc33","country_code":"US","type":"education","lineage":["https://openalex.org/I114027177"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nithin Sivakumaran","raw_affiliation_strings":["University of North Carolina at Chapel Hill, Chapel Hill, USA"],"raw_orcid":"https://orcid.org/0009-0009-8469-0355","affiliations":[{"raw_affiliation_string":"University of North Carolina at Chapel Hill, Chapel Hill, USA","institution_ids":["https://openalex.org/I114027177"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034721319","display_name":"Clayton Cohn","orcid":"https://orcid.org/0000-0003-0856-9587"},"institutions":[{"id":"https://openalex.org/I200719446","display_name":"Vanderbilt University","ror":"https://ror.org/02vm5rt34","country_code":"US","type":"education","lineage":["https://openalex.org/I200719446"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Clayton Cohn","raw_affiliation_strings":["Vanderbilt University, Nashville, USA"],"raw_orcid":"https://orcid.org/0000-0003-0856-9587","affiliations":[{"raw_affiliation_string":"Vanderbilt University, Nashville, USA","institution_ids":["https://openalex.org/I200719446"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037114121","display_name":"Austin Coursey","orcid":"https://orcid.org/0000-0003-1774-6442"},"institutions":[{"id":"https://openalex.org/I200719446","display_name":"Vanderbilt University","ror":"https://ror.org/02vm5rt34","country_code":"US","type":"education","lineage":["https://openalex.org/I200719446"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Austin Coursey","raw_affiliation_strings":["Vanderbilt University, Nashville, USA"],"raw_orcid":"https://orcid.org/0000-0003-1774-6442","affiliations":[{"raw_affiliation_string":"Vanderbilt University, Nashville, USA","institution_ids":["https://openalex.org/I200719446"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059195621","display_name":"Shoubin Yu","orcid":"https://orcid.org/0009-0006-1670-0054"},"institutions":[{"id":"https://openalex.org/I114027177","display_name":"University of North Carolina at Chapel Hill","ror":"https://ror.org/0130frc33","country_code":"US","type":"education","lineage":["https://openalex.org/I114027177"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shoubin Yu","raw_affiliation_strings":["University of North Carolina at Chapel Hill, Chapel Hill, USA"],"raw_orcid":"https://orcid.org/0009-0006-1670-0054","affiliations":[{"raw_affiliation_string":"University of North Carolina at Chapel Hill, Chapel Hill, USA","institution_ids":["https://openalex.org/I114027177"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038276570","display_name":"Elias Stengel-Eskin","orcid":"https://orcid.org/0000-0002-6689-505X"},"institutions":[{"id":"https://openalex.org/I114027177","display_name":"University of North Carolina at Chapel Hill","ror":"https://ror.org/0130frc33","country_code":"US","type":"education","lineage":["https://openalex.org/I114027177"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Elias Stengel-Eskin","raw_affiliation_strings":["University of North Carolina at Chapel Hill, Chapel Hill, USA"],"raw_orcid":"https://orcid.org/0000-0002-6689-505X","affiliations":[{"raw_affiliation_string":"University of North Carolina at Chapel Hill, Chapel Hill, USA","institution_ids":["https://openalex.org/I114027177"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076771617","display_name":"T. S. Ashwin","orcid":null},"institutions":[{"id":"https://openalex.org/I200719446","display_name":"Vanderbilt University","ror":"https://ror.org/02vm5rt34","country_code":"US","type":"education","lineage":["https://openalex.org/I200719446"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ashwin T. S.","raw_affiliation_strings":["Vanderbilt University, Nashville, USA"],"raw_orcid":"https://orcid.org/0000-0002-1690-1626","affiliations":[{"raw_affiliation_string":"Vanderbilt University, Nashville, USA","institution_ids":["https://openalex.org/I200719446"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134616454","display_name":"Mohit Bansal","orcid":"https://orcid.org/0009-0009-2965-5354"},"institutions":[{"id":"https://openalex.org/I114027177","display_name":"University of North Carolina at Chapel Hill","ror":"https://ror.org/0130frc33","country_code":"US","type":"education","lineage":["https://openalex.org/I114027177"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mohit Bansal","raw_affiliation_strings":["University of North Carolina at Chapel Hill, Chapel Hill, USA"],"raw_orcid":"https://orcid.org/0009-0009-2965-5354","affiliations":[{"raw_affiliation_string":"University of North Carolina at Chapel Hill, Chapel Hill, USA","institution_ids":["https://openalex.org/I114027177"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101947700","display_name":"Gautam Biswas","orcid":"https://orcid.org/0000-0003-2236-4027"},"institutions":[{"id":"https://openalex.org/I200719446","display_name":"Vanderbilt University","ror":"https://ror.org/02vm5rt34","country_code":"US","type":"education","lineage":["https://openalex.org/I200719446"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gautam Biswas","raw_affiliation_strings":["Vanderbilt University, Nashville, USA"],"raw_orcid":"https://orcid.org/0000-0002-2752-3878","affiliations":[{"raw_affiliation_string":"Vanderbilt University, Nashville, USA","institution_ids":["https://openalex.org/I200719446"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":9,"corresponding_author_ids":["https://openalex.org/A5076518309"],"corresponding_institution_ids":["https://openalex.org/I200719446"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.95858759,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"536","last_page":"546"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.17800000309944153,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.17800000309944153,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11431","display_name":"Action Observation and Synchronization","score":0.11559999734163284,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10709","display_name":"Social Robot Interaction and HRI","score":0.11500000208616257,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.5300999879837036},{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.49630001187324524},{"id":"https://openalex.org/keywords/multimodality","display_name":"Multimodality","score":0.42340001463890076},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.4171000123023987},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4032000005245209},{"id":"https://openalex.org/keywords/embodied-cognition","display_name":"Embodied cognition","score":0.3880999982357025},{"id":"https://openalex.org/keywords/analytics","display_name":"Analytics","score":0.38769999146461487},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.35850000381469727}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7382000088691711},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.5300999879837036},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.49630001187324524},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.48240000009536743},{"id":"https://openalex.org/C2780910867","wikidata":"https://www.wikidata.org/wiki/Q1952416","display_name":"Multimodality","level":2,"score":0.42340001463890076},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.4171000123023987},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.41510000824928284},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.41370001435279846},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4032000005245209},{"id":"https://openalex.org/C100609095","wikidata":"https://www.wikidata.org/wiki/Q1335050","display_name":"Embodied cognition","level":2,"score":0.3880999982357025},{"id":"https://openalex.org/C79158427","wikidata":"https://www.wikidata.org/wiki/Q485396","display_name":"Analytics","level":2,"score":0.38769999146461487},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3686999976634979},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.35850000381469727},{"id":"https://openalex.org/C78780964","wikidata":"https://www.wikidata.org/wiki/Q7233193","display_name":"Position paper","level":2,"score":0.33079999685287476},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.3138999938964844},{"id":"https://openalex.org/C511192102","wikidata":"https://www.wikidata.org/wiki/Q5156948","display_name":"Comprehension","level":2,"score":0.31150001287460327},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3111000061035156},{"id":"https://openalex.org/C2777601897","wikidata":"https://www.wikidata.org/wiki/Q3409113","display_name":"Presentation (obstetrics)","level":2,"score":0.30399999022483826},{"id":"https://openalex.org/C118147538","wikidata":"https://www.wikidata.org/wiki/Q1126970","display_name":"Metacognition","level":3,"score":0.29820001125335693},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.29159998893737793},{"id":"https://openalex.org/C122783720","wikidata":"https://www.wikidata.org/wiki/Q183065","display_name":"Interpreter","level":2,"score":0.2840000092983246},{"id":"https://openalex.org/C2780660688","wikidata":"https://www.wikidata.org/wiki/Q25052564","display_name":"Multimodal learning","level":2,"score":0.27790001034736633},{"id":"https://openalex.org/C111696304","wikidata":"https://www.wikidata.org/wiki/Q2303697","display_name":"Sorting","level":2,"score":0.2750000059604645},{"id":"https://openalex.org/C2776035091","wikidata":"https://www.wikidata.org/wiki/Q7928819","display_name":"Viewpoints","level":2,"score":0.26010000705718994},{"id":"https://openalex.org/C2779227376","wikidata":"https://www.wikidata.org/wiki/Q6505497","display_name":"Layer (electronics)","level":2,"score":0.2599000036716461}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3785022.3785109","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3785022.3785109","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the LAK26: 16th International Learning Analytics and Knowledge Conference","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3785022.3785109","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3785022.3785109","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the LAK26: 16th International Learning Analytics and Knowledge Conference","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.43788304924964905,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G5643777851","display_name":null,"funder_award_id":"DRL-2112635","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7350814145","display_name":null,"funder_award_id":"R305C240010","funder_id":"https://openalex.org/F4320332210","funder_display_name":"Institute of Education Sciences"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320332210","display_name":"Institute of Education Sciences","ror":"https://ror.org/04et59085"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":36,"referenced_works":["https://openalex.org/W1996643300","https://openalex.org/W2032406331","https://openalex.org/W2117766479","https://openalex.org/W2159065226","https://openalex.org/W2556115049","https://openalex.org/W2744238423","https://openalex.org/W2762345875","https://openalex.org/W2795548522","https://openalex.org/W2942629971","https://openalex.org/W3022118495","https://openalex.org/W3036954260","https://openalex.org/W3136038792","https://openalex.org/W4214890135","https://openalex.org/W4226047775","https://openalex.org/W4311307319","https://openalex.org/W4321442084","https://openalex.org/W4322616103","https://openalex.org/W4327975279","https://openalex.org/W4362720173","https://openalex.org/W4385822293","https://openalex.org/W4388544818","https://openalex.org/W4389325641","https://openalex.org/W4390481633","https://openalex.org/W4391092836","https://openalex.org/W4392484146","https://openalex.org/W4396870811","https://openalex.org/W4398756379","https://openalex.org/W4399488865","https://openalex.org/W4400211188","https://openalex.org/W4401043184","https://openalex.org/W4402671584","https://openalex.org/W4406621251","https://openalex.org/W4407842544","https://openalex.org/W4409740918","https://openalex.org/W4411522868","https://openalex.org/W7127151313"],"related_works":[],"abstract_inverted_index":{"This":[0],"paper":[1],"presents":[2],"an":[3,95,111],"approach":[4],"that":[5,153],"uses":[6],"Large":[7],"Language":[8],"Models":[9],"(LLMs)":[10],"as":[11,189],"late-fusion":[12,191],"interpreters":[13],"to":[14,46,70,88,117],"synthesize":[15],"multimodal":[16,28,194],"signals":[17],"from":[18],"embodied":[19],"classroom":[20,44],"activities":[21],"and":[22,35,49,52,66,76,93,105,128,138,172,197],"infer":[23],"students\u2019":[24,31],"metacognitive":[25],"behaviors.":[26,78],"Our":[27],"pipeline":[29],"analyzes":[30],"movements,":[32],"gaze,":[33],"gestures,":[34],"speech":[36],"within":[37],"a":[38,43,132],"mixed-reality":[39],"simulation":[40],"displayed":[41],"on":[42,166],"screen":[45],"support":[47],"enactment":[48],"learning.":[50],"Vision-":[51],"speech-derived":[53],"features":[54],"are":[55],"fused":[56],"at":[57,100],"the":[58,199],"interpretive":[59],"layer":[60],"via":[61],"zero-shot":[62],"prompting,":[63],"self-consistency":[64],"reasoning,":[65],"targeted":[67],"prompt":[68,129],"engineering":[69],"derive":[71],"planning,":[72],"enacting,":[73],"monitoring,":[74],"reflecting,":[75],"interacting":[77],"We":[79],"investigate":[80],"whether":[81,94],"LLMs":[82,162,188],"can":[83,97,163],"reliably":[84],"integrate":[85],"modality-specific":[86],"analytics":[87,196],"produce":[89],"accurate":[90],"behavioral":[91],"labeling":[92],"LLM-as-a-Judge":[96,116,202],"validate":[98],"them":[99],"scale.":[101],"To":[102],"address":[103],"scalability":[104],"reduce":[106],"human":[107],"burden,":[108],"we":[109,141],"introduce":[110],"automated":[112],"evaluation":[113],"protocol":[114],"employing":[115],"assess":[118],"classification":[119],"quality,":[120],"enabling":[121],"rapid,":[122],"iterative":[123],"benchmarking":[124],"of":[125,135,201],"model":[126],"variants":[127],"strategies.":[130],"Using":[131],"balanced":[133],"corpus":[134],"human-validated":[136],"segments":[137],"perturbed":[139],"controls,":[140],"compare":[142],"text-only":[143],"language":[144],"models":[145,150],"(e.g.,":[146,151],"GPT-5)":[147],"with":[148],"visual\u2013language":[149],"Qwen2.5-VL)":[152],"incorporate":[154],"direct":[155],"visual":[156],"processing.":[157],"Results":[158],"indicate":[159],"late-fusion,":[160],"text-based":[161],"outperform":[164],"VLMs":[165],"behavior":[167],"judgment":[168],"without":[169],"raw":[170],"video,":[171],"precision-":[173],"or":[174,182],"recall-oriented":[175],"prompts":[176],"adjust":[177],"decision":[178],"boundaries":[179],"for":[180,193,203],"subtle":[181],"brief":[183],"segments.":[184],"These":[185],"findings":[186],"position":[187],"effective":[190],"mechanisms":[192],"learning":[195],"demonstrate":[198],"viability":[200],"scalable,":[204],"human-in-the-loop":[205],"evaluation.":[206]},"counts_by_year":[],"updated_date":"2026-04-29T09:16:38.111599","created_date":"2026-04-26T00:00:00"}