{"id":"https://openalex.org/W7134279165","doi":"https://doi.org/10.48550/arxiv.2603.05923","title":"Learning Next Action Predictors from Human-Computer Interaction","display_name":"Learning Next Action Predictors from Human-Computer Interaction","publication_year":2026,"publication_date":"2026-03-06","ids":{"openalex":"https://openalex.org/W7134279165","doi":"https://doi.org/10.48550/arxiv.2603.05923"},"language":null,"primary_location":{"id":"pmh:doi:10.48550/arxiv.2603.05923","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"type":"article","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":null,"any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5128568309","display_name":"Dr. Omar Ahmed Shaikh","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shaikh, Omar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128423475","display_name":"Valentin Teutschbein","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Teutschbein, Valentin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128537813","display_name":"Kanishk Gandhi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gandhi, Kanishk","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128610055","display_name":"Yikun Chi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chi, Yikun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069105490","display_name":"Nick Haber","orcid":"https://orcid.org/0000-0001-8804-7804"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Haber, Nick","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040410474","display_name":"Thomas Robinson","orcid":"https://orcid.org/0000-0001-7097-1599"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Robinson, Thomas","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5121494727","display_name":"Nilam Ram","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ram, Nilam","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053214955","display_name":"Byron Reeves","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Reeves, Byron","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128531031","display_name":"Sherry Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Sherry","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128621225","display_name":"Michael S. Bernstein","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bernstein, Michael S.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5128602796","display_name":"Diyi Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Diyi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.24370731,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12607","display_name":"Personal Information Management and User Behavior","score":0.250900000333786,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12607","display_name":"Personal Information Management and User Behavior","score":0.250900000333786,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.12409999966621399,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.08860000222921371,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.5982999801635742},{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.5857999920845032},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5436999797821045},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.541100025177002},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.5196999907493591},{"id":"https://openalex.org/keywords/similarity","display_name":"Similarity (geometry)","score":0.453000009059906},{"id":"https://openalex.org/keywords/space","display_name":"Space (punctuation)","score":0.3476000130176544},{"id":"https://openalex.org/keywords/parametric-statistics","display_name":"Parametric statistics","score":0.34549999237060547},{"id":"https://openalex.org/keywords/task-analysis","display_name":"Task analysis","score":0.3443000018596649}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7512999773025513},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6011000275611877},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.5982999801635742},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.5857999920845032},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5781000256538391},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5436999797821045},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.541100025177002},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.5196999907493591},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.453000009059906},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3971000015735626},{"id":"https://openalex.org/C2778572836","wikidata":"https://www.wikidata.org/wiki/Q380933","display_name":"Space (punctuation)","level":2,"score":0.3476000130176544},{"id":"https://openalex.org/C117251300","wikidata":"https://www.wikidata.org/wiki/Q1849855","display_name":"Parametric statistics","level":2,"score":0.34549999237060547},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.3443000018596649},{"id":"https://openalex.org/C2778707766","wikidata":"https://www.wikidata.org/wiki/Q202064","display_name":"Phone","level":2,"score":0.3107999861240387},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.3068999946117401},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.2996000051498413},{"id":"https://openalex.org/C2777601683","wikidata":"https://www.wikidata.org/wiki/Q6499736","display_name":"Vocabulary","level":2,"score":0.29840001463890076},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.2921999990940094},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.2816999852657318},{"id":"https://openalex.org/C22019652","wikidata":"https://www.wikidata.org/wiki/Q331309","display_name":"Overfitting","level":3,"score":0.2680000066757202},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.265500009059906},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.26100000739097595},{"id":"https://openalex.org/C197947376","wikidata":"https://www.wikidata.org/wiki/Q5155608","display_name":"Comparability","level":2,"score":0.2606000006198883},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.2599000036716461},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.25760000944137573},{"id":"https://openalex.org/C136389625","wikidata":"https://www.wikidata.org/wiki/Q334384","display_name":"Supervised learning","level":3,"score":0.25600001215934753},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.2513999938964844}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:doi:10.48550/arxiv.2603.05923","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"doi:10.48550/arxiv.2603.05923","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.05923","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:doi:10.48550/arxiv.2603.05923","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","score":0.6473219990730286,"display_name":"Industry, innovation and infrastructure"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Truly":[0],"proactive":[1],"AI":[2],"systems":[3],"must":[4],"anticipate":[5,278],"what":[6,35,244],"we":[7,21,36,82,258,266],"will":[8],"do":[9],"next.":[10],"This":[11,253],"foresight":[12],"demands":[13,28],"far":[14],"richer":[15],"information":[16],"than":[17],"the":[18,31,271],"sparse":[19],"signals":[20],"type":[22],"into":[23],"our":[24],"prompts":[25],"--":[26],"it":[27],"reasoning":[29,154],"over":[30,105,140],"entire":[32],"context":[33,273],"of":[34,51,111,122,165,217,231,237,274],"see":[37],"and":[38,76,103,135,168,193,201],"do.":[39],"We":[40,91,125],"formalize":[41],"this":[42,70,98],"as":[43],"next":[44,66,218,248],"action":[45],"prediction":[46],"(NAP):":[47],"given":[48,156],"a":[49,52,57,129,163,220,245,283],"sequence":[50],"user's":[53,65],"multimodal":[54],"interactions":[55],"with":[56,88,243,286],"computer":[58,86],"(screenshots,":[59],"clicks,":[60],"sensor":[61],"data),":[62],"predict":[63,175],"that":[64,132,268],"action.":[67],"Progress":[68],"on":[69,100,196],"task":[71,285],"requires":[72],"both":[73],"new":[74],"data":[75,198],"modeling":[77],"approaches.":[78],"To":[79],"scale":[80],"data,":[81],"annotate":[83],"longitudinal,":[84],"naturalistic":[85],"use":[87],"vision-language":[89],"models.":[90],"release":[92],"an":[93,179],"open-source":[94],"pipeline":[95],"for":[96],"performing":[97],"labeling":[99],"private":[101],"infrastructure,":[102],"label":[104],"360K":[106],"actions":[107,219],"across":[108,213],"one":[109],"month":[110],"continuous":[112],"phone":[113],"usage":[114],"from":[115,162,270],"20":[116],"users,":[117],"amounting":[118],"to":[119,138,151,174,185,207,255,260,277],"1,800":[120],"hours":[121],"screen":[123],"time.":[124],"then":[126,169],"introduce":[127],"LongNAP,":[128],"user":[130,221,246,275,279],"model":[131],"combines":[133],"parametric":[134],"in-context":[136,173],"learning":[137,269],"reason":[139],"long":[141],"interaction":[142],"histories.":[143],"LongNAP":[144,188,205],"is":[145,227,281],"trained":[146,212],"via":[147],"policy":[148],"gradient":[149],"methods":[150],"generate":[152],"user-specific":[153],"traces":[155,161,172],"some":[157],"context;":[158],"retrieve":[159],"relevant":[160],"library":[164],"past":[166],"traces;":[167],"apply":[170],"retrieved":[171],"future":[176],"actions.":[177],"Using":[178],"LLM-as-judge":[180],"evaluation":[181],"metric":[182],"(0-1":[183],"similarity":[184],"ground":[186],"truth),":[187],"significantly":[189],"outperforms":[190],"supervised":[191],"finetuning":[192],"prompted":[194],"baselines":[195],"held-out":[197],"(by":[199],"79%":[200],"39%":[202],"respectively).":[203],"Additionally,":[204],"generalizes":[206],"held":[208],"out":[209],"users":[210],"when":[211,257],"individuals.":[214],"The":[215],"space":[216],"might":[222],"take":[223],"at":[224],"any":[225],"moment":[226],"unbounded,":[228],"spanning":[229],"thousands":[230],"possible":[232],"outcomes.":[233],"Despite":[234],"this,":[235],"17.1%":[236],"LongNAP's":[238],"predicted":[239],"trajectories":[240],"are":[241],"well-aligned":[242],"does":[247],"(LLM-judge":[249],"score":[250],"$\\geq$":[251],"0.5).":[252],"rises":[254],"26%":[256],"filter":[259],"highly":[261],"confident":[262],"predictions.":[263],"In":[264],"sum,":[265],"argue":[267],"full":[272],"behavior":[276],"needs":[280],"now":[282],"viable":[284],"substantial":[287],"opportunity.":[288]},"counts_by_year":[],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2026-03-10T00:00:00"}