{"id":"https://openalex.org/W7139910887","doi":"https://doi.org/10.48550/arxiv.2603.18988","title":"MERGE: Guided Vision-Language Models for Multi-Actor Event Reasoning and Grounding in Human-Robot Interaction","display_name":"MERGE: Guided Vision-Language Models for Multi-Actor Event Reasoning and Grounding in Human-Robot Interaction","publication_year":2026,"publication_date":"2026-03-19","ids":{"openalex":"https://openalex.org/W7139910887","doi":"https://doi.org/10.48550/arxiv.2603.18988"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.18988","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.18988","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.18988","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5058856575","display_name":"Joerg Deigmoeller","orcid":"https://orcid.org/0009-0007-5931-6973"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Deigmoeller, Joerg","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073021106","display_name":"Nakul Agarwal","orcid":"https://orcid.org/0000-0002-7684-6075"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Agarwal, Nakul","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014423376","display_name":"Stephan Hasler","orcid":"https://orcid.org/0000-0002-3690-3223"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hasler, Stephan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073193182","display_name":"Daniel Tanneberg","orcid":"https://orcid.org/0000-0002-1363-7970"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tanneberg, Daniel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079566434","display_name":"Anna Belardinelli","orcid":"https://orcid.org/0000-0002-0266-3305"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Belardinelli, Anna","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078683360","display_name":"Reza Ghoddoosian","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ghoddoosian, Reza","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130241852","display_name":"Chao Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Chao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054476100","display_name":"Felix Ocker","orcid":"https://orcid.org/0000-0002-5320-2525"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ocker, Felix","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130237525","display_name":"Fan Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Fan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031340269","display_name":"Behzad Dariush","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dariush, Behzad","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5025560073","display_name":"Michael Gienger","orcid":"https://orcid.org/0000-0001-8036-2519"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gienger, Michael","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":11,"corresponding_author_ids":["https://openalex.org/A5058856575"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9833999872207642,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9833999872207642,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10709","display_name":"Social Robot Interaction and HRI","score":0.004399999976158142,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.0019000000320374966,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/merge","display_name":"Merge (version control)","score":0.6388999819755554},{"id":"https://openalex.org/keywords/situation-awareness","display_name":"Situation awareness","score":0.5347999930381775},{"id":"https://openalex.org/keywords/perception","display_name":"Perception","score":0.48069998621940613},{"id":"https://openalex.org/keywords/abstraction","display_name":"Abstraction","score":0.4749000072479248},{"id":"https://openalex.org/keywords/ground","display_name":"Ground","score":0.4390000104904175},{"id":"https://openalex.org/keywords/visual-reasoning","display_name":"Visual reasoning","score":0.39259999990463257},{"id":"https://openalex.org/keywords/situational-ethics","display_name":"Situational ethics","score":0.39010000228881836},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.3562000095844269}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7595999836921692},{"id":"https://openalex.org/C197129107","wikidata":"https://www.wikidata.org/wiki/Q1921621","display_name":"Merge (version control)","level":2,"score":0.6388999819755554},{"id":"https://openalex.org/C145804949","wikidata":"https://www.wikidata.org/wiki/Q478123","display_name":"Situation awareness","level":2,"score":0.5347999930381775},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.48069998621940613},{"id":"https://openalex.org/C124304363","wikidata":"https://www.wikidata.org/wiki/Q673661","display_name":"Abstraction","level":2,"score":0.4749000072479248},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.43950000405311584},{"id":"https://openalex.org/C168993435","wikidata":"https://www.wikidata.org/wiki/Q6501125","display_name":"Ground","level":2,"score":0.4390000104904175},{"id":"https://openalex.org/C2777508537","wikidata":"https://www.wikidata.org/wiki/Q7936620","display_name":"Visual reasoning","level":2,"score":0.39259999990463257},{"id":"https://openalex.org/C9114305","wikidata":"https://www.wikidata.org/wiki/Q1428317","display_name":"Situational ethics","level":2,"score":0.39010000228881836},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.3562000095844269},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.35569998621940613},{"id":"https://openalex.org/C2775945657","wikidata":"https://www.wikidata.org/wiki/Q381442","display_name":"Structuring","level":2,"score":0.33550000190734863},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.32179999351501465},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.31520000100135803},{"id":"https://openalex.org/C2779662365","wikidata":"https://www.wikidata.org/wiki/Q5416694","display_name":"Event (particle physics)","level":2,"score":0.3043000102043152},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.29679998755455017},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.2937999963760376},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.2685999870300293},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.26829999685287476},{"id":"https://openalex.org/C2776544517","wikidata":"https://www.wikidata.org/wiki/Q189447","display_name":"Unexpected events","level":2,"score":0.2632000148296356},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.25690001249313354},{"id":"https://openalex.org/C2781039887","wikidata":"https://www.wikidata.org/wiki/Q1391724","display_name":"Factor (programming language)","level":2,"score":0.2556999921798706},{"id":"https://openalex.org/C157657479","wikidata":"https://www.wikidata.org/wiki/Q2367247","display_name":"Closed captioning","level":3,"score":0.2522999942302704}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.18988","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.18988","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.18988","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.18988","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"We":[0],"introduce":[1,146],"MERGE,":[2],"a":[3,79,82,171,196],"system":[4],"for":[5,142],"situational":[6,25,153],"grounding":[7,168],"of":[8,31,39,49,73,111,125,139,155,173,179,198],"actors,":[9],"objects,":[10],"and":[11,33,35,54,56,93,108,122,132,157,186,202],"events":[12],"in":[13,20],"dynamic":[14],"human-robot":[15,158],"group":[16],"interactions.":[17,66,159],"Effective":[18],"collaboration":[19],"such":[21],"settings":[22],"requires":[23],"consistent":[24],"awareness,":[26],"built":[27],"on":[28],"persistent":[29],"representations":[30],"people":[32],"objects":[34,55],"an":[36],"episodic":[37],"abstraction":[38],"events.":[40],"MERGE":[41,69],"achieves":[42],"this":[43,161],"by":[44,170,195],"uniquely":[45],"identifying":[46],"physical":[47],"instances":[48],"actors":[50],"(humans":[51],"or":[52],"robots)":[53],"structuring":[57],"them":[58],"into":[59],"actor-action-object":[60],"relations,":[61],"ensuring":[62],"temporal":[63],"consistency":[64],"across":[65],"Central":[67],"to":[68,90,130,176],"is":[70],"the":[71,96,105,118,123,137,147,166,177],"integration":[72],"Vision-Language":[74],"Models":[75],"(VLMs)":[76],"guided":[77],"with":[78],"perception":[80],"pipeline:":[81],"lightweight":[83],"streaming":[84],"module":[85],"continuously":[86],"processes":[87],"visual":[88],"input":[89],"detect":[91],"changes":[92],"selectively":[94],"invokes":[95],"VLM":[97],"only":[98],"when":[99],"necessary.":[100],"This":[101],"decoupled":[102],"design":[103],"preserves":[104],"reasoning":[106],"power":[107],"zero-shot":[109],"generalization":[110],"VLMs":[112],"while":[113,191],"improving":[114],"efficiency,":[115],"avoiding":[116],"both":[117],"high":[119],"monetary":[120],"cost":[121],"latency":[124],"frame-by-frame":[126],"captioning":[127],"that":[128],"leads":[129],"fragmented":[131],"delayed":[133],"outputs.":[134],"To":[135],"address":[136],"absence":[138],"suitable":[140],"benchmarks":[141],"multi-actor":[143],"collaboration,":[144],"we":[145],"GROUND":[148],"dataset,":[149,162],"which":[150],"offers":[151],"fine-grained":[152],"annotations":[154],"multi-person":[156],"On":[160],"our":[163],"approach":[164],"improves":[165],"average":[167],"score":[169],"factor":[172,197],"2":[174],"compared":[175],"performance":[178],"VLM-only":[180],"baselines":[181],"-":[182,190],"including":[183],"GPT-4o,":[184],"GPT-5":[185],"Gemini":[187],"2.5":[188],"Flash":[189],"also":[192],"reducing":[193],"run-time":[194],"4.":[199],"The":[200],"code":[201],"data":[203],"are":[204],"available":[205],"at":[206],"www.github.com/HRI-EU/merge.":[207]},"counts_by_year":[],"updated_date":"2026-05-05T08:41:31.759640","created_date":"2026-03-21T00:00:00"}