{"id":"https://openalex.org/W4416065044","doi":"https://doi.org/10.48550/arxiv.2509.07098","title":"Instruction Agent: Enhancing Agent with Expert Demonstration","display_name":"Instruction Agent: Enhancing Agent with Expert Demonstration","publication_year":2025,"publication_date":"2025-09-08","ids":{"openalex":"https://openalex.org/W4416065044","doi":"https://doi.org/10.48550/arxiv.2509.07098"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2509.07098","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2509.07098","pdf_url":"https://arxiv.org/pdf/2509.07098","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2509.07098","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5111319516","display_name":"Yinheng Li","orcid":"https://orcid.org/0009-0000-0233-2760"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Yinheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120323988","display_name":"Hailey Hultquist","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hultquist, Hailey","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066754954","display_name":"Justin Wagle","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wagle, Justin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5084879161","display_name":"Kazuhito Koishida","orcid":"https://orcid.org/0000-0002-3111-5375"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Koishida, Kazuhito","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12607","display_name":"Personal Information Management and User Behavior","score":0.49959999322891235,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12607","display_name":"Personal Information Management and User Behavior","score":0.49959999322891235,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10444","display_name":"Context-Aware Activity Recognition Systems","score":0.07580000162124634,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.049800001084804535,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/bridging","display_name":"Bridging (networking)","score":0.7594000101089478},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6115999817848206},{"id":"https://openalex.org/keywords/graphical-user-interface","display_name":"Graphical user interface","score":0.5634999871253967},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5382999777793884},{"id":"https://openalex.org/keywords/outcome","display_name":"Outcome (game theory)","score":0.4632999897003174},{"id":"https://openalex.org/keywords/interface","display_name":"Interface (matter)","score":0.4564000070095062},{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.4462999999523163},{"id":"https://openalex.org/keywords/user-interface","display_name":"User interface","score":0.421999990940094}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8230999708175659},{"id":"https://openalex.org/C174348530","wikidata":"https://www.wikidata.org/wiki/Q188635","display_name":"Bridging (networking)","level":2,"score":0.7594000101089478},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.6338000297546387},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6115999817848206},{"id":"https://openalex.org/C37789001","wikidata":"https://www.wikidata.org/wiki/Q782543","display_name":"Graphical user interface","level":2,"score":0.5634999871253967},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5382999777793884},{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.4632999897003174},{"id":"https://openalex.org/C113843644","wikidata":"https://www.wikidata.org/wiki/Q901882","display_name":"Interface (matter)","level":4,"score":0.4564000070095062},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.4462999999523163},{"id":"https://openalex.org/C89505385","wikidata":"https://www.wikidata.org/wiki/Q47146","display_name":"User interface","level":2,"score":0.421999990940094},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3610000014305115},{"id":"https://openalex.org/C58328972","wikidata":"https://www.wikidata.org/wiki/Q184609","display_name":"Expert system","level":2,"score":0.31690001487731934},{"id":"https://openalex.org/C32833848","wikidata":"https://www.wikidata.org/wiki/Q4115054","display_name":"Extensibility","level":2,"score":0.3075999915599823},{"id":"https://openalex.org/C41550386","wikidata":"https://www.wikidata.org/wiki/Q529909","display_name":"Multi-agent system","level":2,"score":0.2989000082015991},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.298799991607666},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.29660001397132874},{"id":"https://openalex.org/C2779038628","wikidata":"https://www.wikidata.org/wiki/Q7248497","display_name":"Programming by demonstration","level":3,"score":0.28519999980926514},{"id":"https://openalex.org/C74072328","wikidata":"https://www.wikidata.org/wiki/Q1142726","display_name":"Intelligent agent","level":2,"score":0.28139999508857727},{"id":"https://openalex.org/C5894958","wikidata":"https://www.wikidata.org/wiki/Q2297769","display_name":"Software agent","level":2,"score":0.275299996137619},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2694999873638153},{"id":"https://openalex.org/C66153210","wikidata":"https://www.wikidata.org/wiki/Q5597182","display_name":"Graphical user interface testing","level":4,"score":0.26600000262260437},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.25870001316070557},{"id":"https://openalex.org/C137703981","wikidata":"https://www.wikidata.org/wiki/Q4692093","display_name":"Agent architecture","level":3,"score":0.25769999623298645},{"id":"https://openalex.org/C103683099","wikidata":"https://www.wikidata.org/wiki/Q5370102","display_name":"Embodied agent","level":3,"score":0.25699999928474426},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.25619998574256897}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2509.07098","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2509.07098","pdf_url":"https://arxiv.org/pdf/2509.07098","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2509.07098","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2509.07098","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2509.07098","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2509.07098","pdf_url":"https://arxiv.org/pdf/2509.07098","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Graphical":[0],"user":[1],"interface":[2],"(GUI)":[3],"agents":[4,128,147],"have":[5],"advanced":[6],"rapidly":[7],"but":[8],"still":[9],"struggle":[10],"with":[11],"complex":[12],"tasks":[13,122],"involving":[14],"novel":[15],"UI":[16],"elements,":[17],"long-horizon":[18],"actions,":[19],"and":[20,56,79,98,138,148],"personalized":[21],"trajectories.":[22],"In":[23],"this":[24],"work,":[25],"we":[26],"introduce":[27],"Instruction":[28,111,133],"Agent,":[29],"a":[30,48,114,119,136],"GUI":[31,146,151],"agent":[32,52,75],"that":[33,110,125],"leverages":[34,76],"expert":[35],"demonstrations":[36],"to":[37,83,90,130],"solve":[38],"such":[39],"tasks,":[40],"enabling":[41],"completion":[42],"of":[43,121],"otherwise":[44],"difficult":[45],"workflows.":[46],"Given":[47],"single":[49],"demonstration,":[50],"the":[51,62,66,77,92,142],"extracts":[53],"step-by-step":[54],"instructions":[55],"executes":[57],"them":[58],"by":[59,65],"strictly":[60],"following":[61],"trajectory":[63],"intended":[64],"user,":[67],"which":[68],"avoids":[69],"making":[70],"mistakes":[71],"during":[72,105],"execution.":[73,106],"The":[74,132],"verifier":[78],"backtracker":[80],"modules":[81,87],"further":[82],"improve":[84],"robustness.":[85],"Both":[86],"are":[88],"critical":[89],"understand":[91],"current":[93,145],"outcome":[94],"from":[95],"each":[96],"action":[97],"handle":[99],"unexpected":[100],"interruptions(such":[101],"as":[102],"pop-up":[103],"windows)":[104],"Our":[107],"experiments":[108],"show":[109],"Agent":[112,134],"achieves":[113],"60%":[115],"success":[116],"rate":[117],"on":[118],"set":[120],"in":[123],"OSWorld":[124],"all":[126],"top-ranked":[127],"failed":[129],"complete.":[131],"offers":[135],"practical":[137],"extensible":[139],"framework,":[140],"bridging":[141],"gap":[143],"between":[144],"reliable":[149],"real-world":[150],"task":[152],"automation.":[153]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2025-10-10T00:00:00"}