{"id":"https://openalex.org/W7154157734","doi":"https://doi.org/10.1145/3772318.3790294","title":"The Invisible Mentor: Inferring User Actions from Screen Recordings to Recommend Better Workflows","display_name":"The Invisible Mentor: Inferring User Actions from Screen Recordings to Recommend Better Workflows","publication_year":2026,"publication_date":"2026-04-13","ids":{"openalex":"https://openalex.org/W7154157734","doi":"https://doi.org/10.1145/3772318.3790294"},"language":null,"primary_location":{"id":"doi:10.1145/3772318.3790294","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3772318.3790294","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3772318.3790294","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102911473","display_name":"Litao Yan","orcid":"https://orcid.org/0009-0009-5077-354X"},"institutions":[{"id":"https://openalex.org/I79576946","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10","country_code":"US","type":"education","lineage":["https://openalex.org/I79576946"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Litao Yan","raw_affiliation_strings":["University of Pennsylvania, Philadelphia, PA, USA"],"raw_orcid":"https://orcid.org/0009-0009-5077-354X","affiliations":[{"raw_affiliation_string":"University of Pennsylvania, Philadelphia, PA, USA","institution_ids":["https://openalex.org/I79576946"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003473641","display_name":"Andrew Head","orcid":"https://orcid.org/0000-0002-1523-3347"},"institutions":[{"id":"https://openalex.org/I79576946","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10","country_code":"US","type":"education","lineage":["https://openalex.org/I79576946"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Andrew Head","raw_affiliation_strings":["University of Pennsylvania, Philadelphia, PA, USA"],"raw_orcid":"https://orcid.org/0000-0002-1523-3347","affiliations":[{"raw_affiliation_string":"University of Pennsylvania, Philadelphia, PA, USA","institution_ids":["https://openalex.org/I79576946"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066196993","display_name":"Ken Milne","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ken Milne","raw_affiliation_strings":["Microsoft, Redmond, WA, USA"],"raw_orcid":"https://orcid.org/0009-0004-4796-5026","affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051355395","display_name":"Vu Le","orcid":"https://orcid.org/0000-0003-3727-3291"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Vu Le","raw_affiliation_strings":["Microsoft, Redmond, WA, USA"],"raw_orcid":"https://orcid.org/0000-0003-3727-3291","affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011543162","display_name":"Sumit Gulwani","orcid":"https://orcid.org/0000-0002-9226-9634"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sumit Gulwani","raw_affiliation_strings":["Microsoft, Redmond, WA, USA"],"raw_orcid":"https://orcid.org/0000-0002-9226-9634","affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024034818","display_name":"Chris Parnin","orcid":"https://orcid.org/0000-0001-6182-815X"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chris Parnin","raw_affiliation_strings":["Microsoft, Raleigh, NC, USA"],"raw_orcid":"https://orcid.org/0000-0001-6182-815X","affiliations":[{"raw_affiliation_string":"Microsoft, Raleigh, NC, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5052426710","display_name":"Emerson Murphy-Hill","orcid":"https://orcid.org/0000-0003-3921-9416"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Emerson Murphy-Hill","raw_affiliation_strings":["Microsoft, Mountain View, CA, USA"],"raw_orcid":"https://orcid.org/0000-0003-3921-9416","affiliations":[{"raw_affiliation_string":"Microsoft, Mountain View, CA, USA","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.45872772,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"17"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12607","display_name":"Personal Information Management and User Behavior","score":0.3691999912261963,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12607","display_name":"Personal Information Management and User Behavior","score":0.3691999912261963,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13197","display_name":"Spreadsheets and End-User Computing","score":0.3075000047683716,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.025200000032782555,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.7858999967575073},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6934000253677368},{"id":"https://openalex.org/keywords/limiting","display_name":"Limiting","score":0.5519000291824341},{"id":"https://openalex.org/keywords/task-analysis","display_name":"Task analysis","score":0.33320000767707825},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.3131999969482422},{"id":"https://openalex.org/keywords/user-interface","display_name":"User interface","score":0.2630999982357025}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.788100004196167},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.7858999967575073},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6934000253677368},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.5895000100135803},{"id":"https://openalex.org/C188198153","wikidata":"https://www.wikidata.org/wiki/Q1613840","display_name":"Limiting","level":2,"score":0.5519000291824341},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4133000075817108},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.33320000767707825},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.3131999969482422},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.289000004529953},{"id":"https://openalex.org/C89505385","wikidata":"https://www.wikidata.org/wiki/Q47146","display_name":"User interface","level":2,"score":0.2630999982357025},{"id":"https://openalex.org/C2778751112","wikidata":"https://www.wikidata.org/wiki/Q835016","display_name":"Window (computing)","level":2,"score":0.262800008058548},{"id":"https://openalex.org/C48677424","wikidata":"https://www.wikidata.org/wiki/Q6888088","display_name":"Mode (computer interface)","level":2,"score":0.26249998807907104},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.26190000772476196},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2574000060558319}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3772318.3790294","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3772318.3790294","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2509.26557","is_oa":true,"landing_page_url":"https://arxiv.org/abs/2509.26557","pdf_url":"https://arxiv.org/pdf/2509.26557","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3772318.3790294","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3772318.3790294","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":48,"referenced_works":["https://openalex.org/W1674092947","https://openalex.org/W1857789879","https://openalex.org/W1978240628","https://openalex.org/W2036433269","https://openalex.org/W2036925928","https://openalex.org/W2038550763","https://openalex.org/W2051493940","https://openalex.org/W2055821246","https://openalex.org/W2059216172","https://openalex.org/W2060966585","https://openalex.org/W2102970299","https://openalex.org/W2109228456","https://openalex.org/W2111009050","https://openalex.org/W2112726785","https://openalex.org/W2116041277","https://openalex.org/W2116436752","https://openalex.org/W2121867457","https://openalex.org/W2141613588","https://openalex.org/W2146105230","https://openalex.org/W2161052636","https://openalex.org/W2167213883","https://openalex.org/W2272645513","https://openalex.org/W2293101314","https://openalex.org/W2296648354","https://openalex.org/W2413573154","https://openalex.org/W2414359420","https://openalex.org/W2795734224","https://openalex.org/W2884547357","https://openalex.org/W2898830973","https://openalex.org/W2953440730","https://openalex.org/W2956017828","https://openalex.org/W3086532961","https://openalex.org/W3111869873","https://openalex.org/W4287887993","https://openalex.org/W4300325272","https://openalex.org/W4366548330","https://openalex.org/W4384345707","https://openalex.org/W4388483081","https://openalex.org/W4391833165","https://openalex.org/W4396833044","https://openalex.org/W4399156370","https://openalex.org/W4402410118","https://openalex.org/W4404781885","https://openalex.org/W4408615011","https://openalex.org/W4409736272","https://openalex.org/W4410636877","https://openalex.org/W4413982823","https://openalex.org/W4415250343"],"related_works":[],"abstract_inverted_index":{"Users":[0],"of":[1],"feature-rich":[2],"tools":[3],"like":[4],"Excel":[5],"often":[6],"miss":[7],"more":[8,103,105,108,121],"efficient":[9],"workflows,":[10],"repeating":[11],"tedious":[12],"steps":[13],"and":[14,76,107],"making":[15],"avoidable":[16],"errors.":[17],"Current":[18],"approaches":[19],"to":[20,58,73,84],"helping":[21],"them":[22,56],"require":[23],"either":[24],"manual":[25],"prompting,":[26],"which":[27,35],"is":[28,36],"effortful":[29],"for":[30,38],"users,":[31],"or":[32],"automated":[33],"logging,":[34],"limiting":[37],"developers.":[39],"We":[40],"present":[41],"InvisibleMentor,":[42],"a":[43,70,80,96,113],"system":[44],"inspired":[45],"by":[46],"over-the-shoulder":[47],"learning:":[48],"it":[49,60],"observes":[50],"what":[51],"users":[52,128],"do,":[53],"then":[54,78],"shows":[55],"how":[57],"do":[59,63,120],"better.":[61],"To":[62],"this,":[64],"InvisibleMentor":[65],"analyzes":[66],"screen":[67],"recordings":[68],"with":[69],"vision-language":[71],"model":[72,83],"reconstruct":[74],"actions":[75],"context,":[77],"uses":[79],"large":[81],"language":[82],"generate":[85],"vision-grounded":[86],"task":[87],"reflection,":[88],"structured":[89],"suggestions":[90,102],"grounded":[91],"in":[92],"observed":[93],"behavior.":[94],"In":[95],"user":[97],"study,":[98],"participants":[99],"found":[100],"InvisibleMentor\u2019s":[101],"clear,":[104],"relevant,":[106],"useful":[109],"than":[110,122],"those":[111],"from":[112],"prompt-based":[114],"assistant,":[115],"demonstrating":[116],"that":[117],"AI":[118],"can":[119,126],"automate":[123],"away":[124],"work\u2014it":[125],"help":[127],"master":[129],"it.":[130]},"counts_by_year":[],"updated_date":"2026-07-03T08:13:44.112507","created_date":"2026-04-14T00:00:00"}