{"id":"https://openalex.org/W7140275094","doi":"https://doi.org/10.48550/arxiv.2603.22321","title":"From Instructions to Assistance: a Dataset Aligning Instruction Manuals with Assembly Videos for Evaluating Multimodal LLMs","display_name":"From Instructions to Assistance: a Dataset Aligning Instruction Manuals with Assembly Videos for Evaluating Multimodal LLMs","publication_year":2026,"publication_date":"2026-03-20","ids":{"openalex":"https://openalex.org/W7140275094","doi":"https://doi.org/10.48550/arxiv.2603.22321"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.22321","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.22321","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.22321","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5061447676","display_name":"Federico Toschi","orcid":"https://orcid.org/0000-0001-5935-2332"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Toschi, Federico","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130619214","display_name":"Nicol\u00f2 Brunello","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Brunello, Nicol\u00f2","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057059969","display_name":"Andrea Sassella","orcid":"https://orcid.org/0000-0001-9067-8496"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sassella, Andrea","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034545062","display_name":"Vincenzo Scotti","orcid":"https://orcid.org/0000-0002-8765-604X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Scotti, Vincenzo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5069487278","display_name":"Mark Carman","orcid":"https://orcid.org/0000-0001-6575-9737"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Carman, Mark James","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.5821999907493591,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.5821999907493591,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.06750000268220901,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.05299999937415123,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5282999873161316},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.5188999772071838},{"id":"https://openalex.org/keywords/point","display_name":"Point (geometry)","score":0.48899999260902405},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.477400004863739},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4327000081539154},{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.3752000033855438},{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.3684999942779541},{"id":"https://openalex.org/keywords/continuation","display_name":"Continuation","score":0.36410000920295715}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7871999740600586},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.5307999849319458},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5282999873161316},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.5188999772071838},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.48899999260902405},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.477400004863739},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4512999951839447},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4327000081539154},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.3752000033855438},{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.3684999942779541},{"id":"https://openalex.org/C88626702","wikidata":"https://www.wikidata.org/wiki/Q1128903","display_name":"Continuation","level":2,"score":0.36410000920295715},{"id":"https://openalex.org/C153715457","wikidata":"https://www.wikidata.org/wiki/Q254183","display_name":"Augmented reality","level":2,"score":0.34779998660087585},{"id":"https://openalex.org/C25343380","wikidata":"https://www.wikidata.org/wiki/Q277521","display_name":"Relation (database)","level":2,"score":0.34369999170303345},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3310999870300293},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.32420000433921814},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3116999864578247},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.30709999799728394},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.29840001463890076},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.2948000133037567},{"id":"https://openalex.org/C2779965156","wikidata":"https://www.wikidata.org/wiki/Q5227350","display_name":"Data sharing","level":3,"score":0.2806999981403351},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2606000006198883},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.2603999972343445},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.2549999952316284},{"id":"https://openalex.org/C58489278","wikidata":"https://www.wikidata.org/wiki/Q1172284","display_name":"Data set","level":2,"score":0.2533999979496002},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2522999942302704},{"id":"https://openalex.org/C482391","wikidata":"https://www.wikidata.org/wiki/Q101244","display_name":"Acronym","level":2,"score":0.25060001015663147}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.22321","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.22321","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.22321","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.22321","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"The":[0],"recent":[1],"advancements":[2],"introduced":[3],"by":[4,161,244],"Large":[5,35],"Language":[6,36],"Models":[7,37],"(LLMs)":[8],"have":[9],"transformed":[10],"how":[11],"Artificial":[12],"Intelligence":[13],"(AI)":[14],"can":[15,188,222],"support":[16],"complex,":[17],"real":[18,78],"world":[19],"tasks,":[20,83],"pushing":[21],"research":[22],"outside":[23],"the":[24,40,54,63,89,92,101,118,121,132,167,183,193,213,226,250],"text":[25,257],"boundaries":[26],"towards":[27],"multi":[28,253],"modal":[29],"contexts":[30],"and":[31,164,219,246,255],"leading":[32],"to":[33,61,115,139,169,177,180,191,211,225],"Multimodal":[34],"(MLMs).":[38],"Given":[39],"current":[41],"adoption":[42],"of":[43,57,66,88,104,134,143,156,186,215],"LLM":[44],"based":[45],"assistants":[46,68,80],"in":[47,81],"solving":[48],"technical":[49,146],"or":[50,97,110],"domain":[51],"specific":[52],"problems,":[53],"natural":[55],"continuation":[56],"this":[58,126,141,149,175],"trend":[59],"is":[60,123,242],"extend":[62],"input":[64],"domains":[65],"these":[67,72],"exploiting":[69],"MLMs.":[70],"Ideally,":[71],"MLMs":[73,138,187,208,221],"should":[74],"be":[75,189],"used":[76,174,190],"as":[77],"time":[79],"procedural":[82,238],"hopefully":[84],"integrating":[85],"a":[86,153],"view":[87,105],"environment":[90],"where":[91],"user":[93,122],"being":[94],"assisted":[95],"is,":[96],"even":[98],"better":[99],"sharing":[100],"same":[102,119],"point":[103],"via":[106],"Virtual":[107],"Reality":[108,112],"(VR)":[109],"Augmented":[111],"(AR)":[113],"supports,":[114],"reason":[116],"over":[117],"scenario":[120],"experiencing.":[124],"With":[125],"work,":[127],"we":[128,151],"aim":[129],"at":[130],"evaluating":[131],"quality":[133],"currently":[135],"openly":[136],"available":[137],"provide":[140],"kind":[142],"assistance":[144],"on":[145],"tasks.":[147],"To":[148],"end,":[150],"annotated":[152],"data":[154],"set":[155],"furniture":[157],"assembly":[158,216],"with":[159],"step":[160,162],"labels":[163],"manual":[165,228],"references:":[166],"Manual":[168],"Action":[170],"Dataset":[171],"(M2AD).":[172],"We":[173],"dataset":[176],"assess":[178],"(1)":[179],"which":[181],"extent":[182],"reasoning":[184],"abilities":[185],"reduce":[192],"need":[194,251],"for":[195,199,252],"detailed":[196],"labelling,":[197],"allowing":[198],"more":[200],"efficient,":[201],"cost":[202],"effective":[203],"annotation":[204],"practices,":[205],"(2)":[206],"whether":[207,220],"are":[209],"able":[210],"track":[212],"progression":[214],"steps":[217],"(3)":[218],"refer":[223],"correctly":[224],"instruction":[227],"pages.":[229],"Our":[230],"results":[231],"showed":[232],"that":[233],"while":[234],"some":[235],"models":[236],"understand":[237],"sequences,":[239],"their":[240],"performance":[241],"limited":[243],"architectural":[245],"hardware":[247],"constraints,":[248],"highlighting":[249],"image":[254,258],"interleaved":[256],"reasoning.":[259]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-03-26T00:00:00"}