{"id":"https://openalex.org/W7202393714","doi":"https://doi.org/10.64898/2026.08.09.26360057","title":"Natural-language retrieval with multimodal embeddings identifies candidate developmental behaviors in caregiver-child recordings","display_name":"Natural-language retrieval with multimodal embeddings identifies candidate developmental behaviors in caregiver-child recordings","publication_year":2026,"publication_date":"2026-08-13","ids":{"openalex":"https://openalex.org/W7202393714","doi":"https://doi.org/10.64898/2026.08.09.26360057"},"language":"en","primary_location":{"id":"doi:10.64898/2026.08.09.26360057","doi":"https://doi.org/10.64898/2026.08.09.26360057","is_oa":true,"landing_page_url":"https://doi.org/10.64898/2026.08.09.26360057","pdf_url":null,"source":{"id":"https://openalex.org/S3005729997","display_name":"medRxiv","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":["https://openalex.org/P4310315909"],"host_organization_lineage_names":["Cold Spring Harbor Laboratory Press"],"type":"repository"},"license":"cc-by-nd","license_id":"https://openalex.org/licenses/cc-by-nd","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"medRxiv","raw_type":"posted-content"},"volume":null,"type":"preprint","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.64898/2026.08.09.26360057","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5032357963","display_name":"Benson Mwangi","orcid":"https://orcid.org/0000-0002-1717-4395","observed_orcids":["https://orcid.org/0000-0002-1717-4395"]},"institutions":[{"id":"https://openalex.org/I919571938","display_name":"The University of Texas Health Science Center at Houston","ror":"https://ror.org/03gds6c39","country_code":"US","type":"education","lineage":["https://openalex.org/I919571938"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Benson Mwangi","raw_affiliation_strings":["UT Center of Excellence on Mood Disorders, Louis A. Faillace, MD, Department of Psychiatry and Behavioral Sciences, School of Behavioral Health Sciences, The University of Texas Health Science Center at Houston, Houston, TX, USA"],"raw_orcid":"https://orcid.org/0000-0002-1717-4395","affiliations":[{"raw_affiliation_string":"UT Center of Excellence on Mood Disorders, Louis A. Faillace, MD, Department of Psychiatry and Behavioral Sciences, School of Behavioral Health Sciences, The University of Texas Health Science Center at Houston, Houston, TX, USA","institution_ids":["https://openalex.org/I919571938"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006756404","display_name":"Mon\u2010Ju Wu","orcid":"https://orcid.org/0000-0002-0540-8027","observed_orcids":["https://orcid.org/0000-0002-0540-8027"]},"institutions":[{"id":"https://openalex.org/I919571938","display_name":"The University of Texas Health Science Center at Houston","ror":"https://ror.org/03gds6c39","country_code":"US","type":"education","lineage":["https://openalex.org/I919571938"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mon-Ju Wu","raw_affiliation_strings":["UT Center of Excellence on Mood Disorders, Louis A. Faillace, MD, Department of Psychiatry and Behavioral Sciences, School of Behavioral Health Sciences, The University of Texas Health Science Center at Houston, Houston, TX, USA"],"raw_orcid":"https://orcid.org/0000-0002-0540-8027","affiliations":[{"raw_affiliation_string":"UT Center of Excellence on Mood Disorders, Louis A. Faillace, MD, Department of Psychiatry and Behavioral Sciences, School of Behavioral Health Sciences, The University of Texas Health Science Center at Houston, Houston, TX, USA","institution_ids":["https://openalex.org/I919571938"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045977979","display_name":"Rosleen Mansour","orcid":"https://orcid.org/0000-0001-9139-5316","observed_orcids":["https://orcid.org/0000-0001-9139-5316"]},"institutions":[{"id":"https://openalex.org/I919571938","display_name":"The University of Texas Health Science Center at Houston","ror":"https://ror.org/03gds6c39","country_code":"US","type":"education","lineage":["https://openalex.org/I919571938"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rosleen Mansour","raw_affiliation_strings":["Louis A. Faillace, MD, Department of Psychiatry and Behavioral Sciences, McGovern Medical School at UTHealth Houston, Houston, TX 77054, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Louis A. Faillace, MD, Department of Psychiatry and Behavioral Sciences, McGovern Medical School at UTHealth Houston, Houston, TX 77054, USA","institution_ids":["https://openalex.org/I919571938"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026659174","display_name":"Gabriel Anzueto","orcid":"https://orcid.org/0009-0004-4955-7595","observed_orcids":["https://orcid.org/0009-0004-4955-7595"]},"institutions":[{"id":"https://openalex.org/I919571938","display_name":"The University of Texas Health Science Center at Houston","ror":"https://ror.org/03gds6c39","country_code":"US","type":"education","lineage":["https://openalex.org/I919571938"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gabriel Anzueto","raw_affiliation_strings":["Dan L Duncan Children\u2019s, Neurodevelopmental Center at the Children\u2019s Learning Institute, Department of Pediatrics, McGovern Medical School at the University of Texas, Health Science Center at Houston, Houston, Texas, USA"],"raw_orcid":"https://orcid.org/0009-0004-4955-7595","affiliations":[{"raw_affiliation_string":"Dan L Duncan Children\u2019s, Neurodevelopmental Center at the Children\u2019s Learning Institute, Department of Pediatrics, McGovern Medical School at the University of Texas, Health Science Center at Houston, Houston, Texas, USA","institution_ids":["https://openalex.org/I919571938"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5086070617","display_name":"Antonio F. Pag\u00e1n","orcid":"https://orcid.org/0000-0002-4724-768X","observed_orcids":["https://orcid.org/0000-0002-4724-768X","https://orcid.org/0009-0003-2015-1906"]},"institutions":[{"id":"https://openalex.org/I919571938","display_name":"The University of Texas Health Science Center at Houston","ror":"https://ror.org/03gds6c39","country_code":"US","type":"education","lineage":["https://openalex.org/I919571938"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Antonio F. Pag\u00e1n","raw_affiliation_strings":["Louis A. Faillace, MD, Department of Psychiatry and Behavioral Sciences, McGovern Medical School at UTHealth Houston, Houston, TX 77054, USA"],"raw_orcid":"https://orcid.org/0000-0002-4724-768X","affiliations":[{"raw_affiliation_string":"Louis A. Faillace, MD, Department of Psychiatry and Behavioral Sciences, McGovern Medical School at UTHealth Houston, Houston, TX 77054, USA","institution_ids":["https://openalex.org/I919571938"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I919571938"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.44530001282691956,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.44530001282691956,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10730","display_name":"Language Development and Disorders","score":0.31839999556541443,"subfield":{"id":"https://openalex.org/subfields/3204","display_name":"Developmental and Educational Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.027799999341368675,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/natural-language-retrieval","display_name":"natural language retrieval","score":0.8960000276565552},{"id":"https://openalex.org/keywords/multimodal-embeddings","display_name":"multimodal embeddings","score":0.9959999918937683},{"id":"https://openalex.org/keywords/caregiver-child-interaction","display_name":"caregiver-child interaction","score":0.7950000166893005},{"id":"https://openalex.org/keywords/developmental-behavior","display_name":"developmental behavior","score":0.5180000066757202},{"id":"https://openalex.org/keywords/evidence-selection","display_name":"evidence selection","score":0.5789999961853027},{"id":"https://openalex.org/keywords/vocabulary-size","display_name":"vocabulary size","score":0.6309999823570251},{"id":"https://openalex.org/keywords/rare-event-detection","display_name":"rare event detection","score":0.574999988079071},{"id":"https://openalex.org/keywords/pointing","display_name":"pointing","score":0.44999998807907104},{"id":"https://openalex.org/keywords/babbling","display_name":"babbling","score":0.9760000109672546},{"id":"https://openalex.org/keywords/speech-and-gesture","display_name":"speech and gesture","score":0.3440000116825104}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7347000241279602},{"id":"https://openalex.org/C2777601683","wikidata":"https://www.wikidata.org/wiki/Q6499736","display_name":"Vocabulary","level":2,"score":0.5648999810218811},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.5467000007629395},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.49050000309944153},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.4593000113964081},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.4027999937534332},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3662000000476837},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.32850000262260437},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.2937000095844269},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.25949999690055847}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.64898/2026.08.09.26360057","doi":"https://doi.org/10.64898/2026.08.09.26360057","is_oa":true,"landing_page_url":"https://doi.org/10.64898/2026.08.09.26360057","pdf_url":null,"source":{"id":"https://openalex.org/S3005729997","display_name":"medRxiv","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":["https://openalex.org/P4310315909"],"host_organization_lineage_names":["Cold Spring Harbor Laboratory Press"],"type":"repository"},"license":"cc-by-nd","license_id":"https://openalex.org/licenses/cc-by-nd","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"medRxiv","raw_type":"posted-content"}],"best_oa_location":{"id":"doi:10.64898/2026.08.09.26360057","doi":"https://doi.org/10.64898/2026.08.09.26360057","is_oa":true,"landing_page_url":"https://doi.org/10.64898/2026.08.09.26360057","pdf_url":null,"source":{"id":"https://openalex.org/S3005729997","display_name":"medRxiv","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":["https://openalex.org/P4310315909"],"host_organization_lineage_names":["Cold Spring Harbor Laboratory Press"],"type":"repository"},"license":"cc-by-nd","license_id":"https://openalex.org/licenses/cc-by-nd","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"medRxiv","raw_type":"posted-content"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":61,"referenced_works":["https://openalex.org/W31633506","https://openalex.org/W389331340","https://openalex.org/W1593934207","https://openalex.org/W1596515083","https://openalex.org/W1964575515","https://openalex.org/W1999465661","https://openalex.org/W2036439084","https://openalex.org/W2069870183","https://openalex.org/W2089702058","https://openalex.org/W2116739230","https://openalex.org/W2123081516","https://openalex.org/W2142798752","https://openalex.org/W2144345994","https://openalex.org/W2149860264","https://openalex.org/W2162994165","https://openalex.org/W2181523240","https://openalex.org/W2191828469","https://openalex.org/W2259559558","https://openalex.org/W2512902140","https://openalex.org/W2535114562","https://openalex.org/W2605959375","https://openalex.org/W2806811344","https://openalex.org/W2903031159","https://openalex.org/W2910821810","https://openalex.org/W2939803422","https://openalex.org/W2970963327","https://openalex.org/W2976851556","https://openalex.org/W2989026243","https://openalex.org/W2995296037","https://openalex.org/W3003257820","https://openalex.org/W3035965352","https://openalex.org/W3045733287","https://openalex.org/W3082004699","https://openalex.org/W3126524703","https://openalex.org/W3135367836","https://openalex.org/W3194728950","https://openalex.org/W3203737321","https://openalex.org/W4213009331","https://openalex.org/W4213250451","https://openalex.org/W4229007573","https://openalex.org/W4280488363","https://openalex.org/W4285037000","https://openalex.org/W4367174514","https://openalex.org/W4378713906","https://openalex.org/W4386071707","https://openalex.org/W4387252268","https://openalex.org/W4387389147","https://openalex.org/W4392945187","https://openalex.org/W4401753915","https://openalex.org/W4402480327","https://openalex.org/W4402703993","https://openalex.org/W4403780684","https://openalex.org/W4413395722","https://openalex.org/W4415016763","https://openalex.org/W4416003979","https://openalex.org/W4416830667","https://openalex.org/W4417064428","https://openalex.org/W7160916879","https://openalex.org/W7165568527","https://openalex.org/W7167580465","https://openalex.org/W7212726428"],"related_works":["https://openalex.org/W7215901343","https://openalex.org/W2946378513","https://openalex.org/W4304892597","https://openalex.org/W7215032987","https://openalex.org/W202709454","https://openalex.org/W7166806903","https://openalex.org/W2404363352","https://openalex.org/W3095548359","https://openalex.org/W7130946112","https://openalex.org/W4401043675"],"abstract_inverted_index":{"Abstract":[0],"Background":[1],"Naturalistic":[2],"audiovisual":[3],"recordings":[4,80],"of":[5,86,115,192,282],"caregiver-child":[6,79,216],"interactions":[7],"contain":[8],"rich":[9],"developmental":[10,209],"signals.":[11],"However,":[12,184,218],"extracting":[13],"interpretable":[14],"clinical":[15,298],"measures":[16],"requires":[17],"resource-intensive":[18],"manual":[19,273],"coding.":[20],"To":[21],"address":[22],"this":[23],"bottleneck,":[24],"we":[25,155],"evaluated":[26],"natural-language":[27,59],"queries":[28],"for":[29,58,130,134,255,262,272],"retrieving":[30],"specific":[31],"behavioral":[32,75,274],"moments":[33],"from":[34,62,88],"these":[35,219,249,287],"recordings,":[36],"applying":[37],"multimodal":[38],"embeddings":[39,204],"as":[40,141,268],"an":[41,269],"automated":[42,256],"evidence-selection":[43,257],"layer.":[44],"Methods":[45],"We":[46,70,246],"compared":[47],"three":[48],"embedding":[49],"models":[50,250,288],"(Jina":[51],"Embeddings":[52,101],"v5":[53,102],"Omni,":[54],"LanguageBind,":[55],"and":[56,64,83,119,137,143,176,181,211,285],"Wave7B)":[57],"retrieval":[60,108,125],"directly":[61],"audio":[63,153],"video":[65],"streams,":[66],"bypassing":[67],"transcript":[68],"text.":[69],"assessed":[71],"performance":[72],"across":[73,289],"27":[74],"targets":[76,132],"in":[77],"277":[78],"(14,":[81],"24,":[82,180],"36":[84,182],"months":[85],"age)":[87],"the":[89,105,147,151,161,185,189,199,280],"Early":[90],"Head":[91],"Start":[92],"Talkbank":[93],"corpus,":[94],"yielding":[95],"7,479":[96],"recording-target":[97],"queries.":[98],"Results":[99],"Jina":[100,158],"Omni":[103],"achieved":[104],"highest":[106],"top-10":[107],"success":[109],"(text-to-audio":[110],"38.3%;":[111],"text-to-video":[112],"36.4%),":[113],"ahead":[114],"LanguageBind":[116],"(37.0%;":[117],"34.5%)":[118],"Wave7B":[120],"(36.1%;":[121],"35.0%).":[122],"Across":[123],"models,":[124],"was":[126],"substantially":[127],"more":[128],"successful":[129],"common":[131],"than":[133,266],"rare":[135,225],"vocal":[136],"gestural":[138],"behaviors,":[139],"such":[140],"pointing":[142],"babbling.":[144],"By":[145],"analyzing":[146],"spoken":[148],"words":[149,194],"within":[150,214],"retrieved":[152],"clips,":[154],"found":[156],"that":[157,248],"accurately":[159,231],"ranked":[160],"children":[162,233],"by":[163,234],"their":[164],"relative":[165,235],"vocabulary":[166,236],"size":[167],"at":[168,178],"each":[169,195],"age":[170],"(Spearman":[171],"\u2374":[172],"=":[173],"0.68,":[174],"0.82,":[175],"0.90":[177],"14,":[179],"months).":[183],"model":[186],"severely":[187],"underestimated":[188],"total":[190],"number":[191],"unique":[193],"child":[196],"used":[197],"throughout":[198],"full":[200],"session.":[201],"Conclusion":[202],"Multimodal":[203],"can":[205,230],"successfully":[206],"pinpoint":[207],"important":[208],"behaviors":[210,284],"speech":[212],"patterns":[213],"lengthy":[215],"recordings.":[217],"systems":[220],"still":[221],"struggle":[222],"to":[223,240,258],"locate":[224],"events.":[226],"Additionally,":[227],"while":[228],"they":[229,238],"rank":[232],"size,":[237],"fail":[239],"measure":[241],"a":[242],"child\u2019s":[243],"complete":[244],"vocabulary.":[245],"conclude":[247],"are":[251,292],"currently":[252],"best":[253],"suited":[254],"prioritize":[259],"relevant":[260],"segments":[261],"expert":[263],"interpretation":[264],"rather":[265],"acting":[267],"independent":[270],"replacement":[271],"coding":[275],"or":[276],"language":[277],"assessment.":[278],"Improving":[279],"detection":[281],"infrequent":[283],"validating":[286],"external":[290],"datasets":[291],"essential":[293],"next":[294],"steps":[295],"before":[296],"real-world":[297],"deployment.":[299]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-08-14T00:00:00"}