{"id":"https://openalex.org/W6906686483","doi":"https://doi.org/10.18419/opus-15036","title":"Leveraging large language models for latent intention recognition and next action prediction","display_name":"Leveraging large language models for latent intention recognition and next action prediction","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W6906686483","doi":"https://doi.org/10.18419/opus-15036"},"language":"en","primary_location":{"id":"pmh:oai:elib.uni-stuttgart.de:11682/15055","doi":null,"is_oa":true,"landing_page_url":"http://nbn-resolving.de/urn:nbn:de:bsz:93-opus-ds-150552","pdf_url":null,"source":{"id":"https://openalex.org/S4306401556","display_name":"OPUS Publication Server of the University of Stuttgart (University of Stuttgart)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I100066346","host_organization_name":"University of Stuttgart","host_organization_lineage":["https://openalex.org/I100066346"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"masterThesis"},"volume":null,"type":"dissertation","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://nbn-resolving.de/urn:nbn:de:bsz:93-opus-ds-150552","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Ahmed, Mohamed","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Ahmed, Mohamed","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.21979999542236328,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.21979999542236328,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10470","display_name":"Usability and User Interface Design","score":0.15469999611377716,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.11429999768733978,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/next-action-prediction","display_name":"next action prediction","score":0.9800000190734863},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.847000002861023},{"id":"https://openalex.org/keywords/graphical-user-interface","display_name":"graphical user interface","score":0.9190000295639038},{"id":"https://openalex.org/keywords/autonomous-agents","display_name":"autonomous agents","score":0.703000009059906},{"id":"https://openalex.org/keywords/prompt-engineering","display_name":"prompt engineering","score":0.9700000286102295},{"id":"https://openalex.org/keywords/task-automation","display_name":"task automation","score":0.7139999866485596},{"id":"https://openalex.org/keywords/motifs","display_name":"motifs","score":0.9620000123977661}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8090999722480774},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.7225000262260437},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.5587999820709229},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.5530999898910522},{"id":"https://openalex.org/C37789001","wikidata":"https://www.wikidata.org/wiki/Q782543","display_name":"Graphical user interface","level":2,"score":0.5292999744415283},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5164999961853027},{"id":"https://openalex.org/C89505385","wikidata":"https://www.wikidata.org/wiki/Q47146","display_name":"User interface","level":2,"score":0.46160000562667847},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.46149998903274536},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.4480000138282776},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.43869999051094055},{"id":"https://openalex.org/C67712803","wikidata":"https://www.wikidata.org/wiki/Q7901853","display_name":"User modeling","level":3,"score":0.4375},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.35420000553131104},{"id":"https://openalex.org/C2987834672","wikidata":"https://www.wikidata.org/wiki/Q4677630","display_name":"Action recognition","level":3,"score":0.35339999198913574},{"id":"https://openalex.org/C66153210","wikidata":"https://www.wikidata.org/wiki/Q5597182","display_name":"Graphical user interface testing","level":4,"score":0.35179999470710754},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.3490999937057495},{"id":"https://openalex.org/C201025465","wikidata":"https://www.wikidata.org/wiki/Q11248500","display_name":"User experience design","level":2,"score":0.2989000082015991},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2802000045776367},{"id":"https://openalex.org/C155846161","wikidata":"https://www.wikidata.org/wiki/Q1143367","display_name":"Graphical model","level":2,"score":0.2694000005722046},{"id":"https://openalex.org/C149229913","wikidata":"https://www.wikidata.org/wiki/Q135707","display_name":"User interface design","level":3,"score":0.2655999958515167},{"id":"https://openalex.org/C2983448237","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Language understanding","level":2,"score":0.2502000033855438}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:elib.uni-stuttgart.de:11682/15055","doi":null,"is_oa":true,"landing_page_url":"http://nbn-resolving.de/urn:nbn:de:bsz:93-opus-ds-150552","pdf_url":null,"source":{"id":"https://openalex.org/S4306401556","display_name":"OPUS Publication Server of the University of Stuttgart (University of Stuttgart)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I100066346","host_organization_name":"University of Stuttgart","host_organization_lineage":["https://openalex.org/I100066346"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"masterThesis"},{"id":"doi:10.18419/opus-15036","doi":"https://doi.org/10.18419/opus-15036","is_oa":true,"landing_page_url":"https://doi.org/10.18419/opus-15036","pdf_url":null,"source":{"id":"https://openalex.org/S7407052998","display_name":"Universit\u00e4tsbibliothek Stuttgart","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Collection"}],"best_oa_location":{"id":"pmh:oai:elib.uni-stuttgart.de:11682/15055","doi":null,"is_oa":true,"landing_page_url":"http://nbn-resolving.de/urn:nbn:de:bsz:93-opus-ds-150552","pdf_url":null,"source":{"id":"https://openalex.org/S4306401556","display_name":"OPUS Publication Server of the University of Stuttgart (University of Stuttgart)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I100066346","host_organization_name":"University of Stuttgart","host_organization_lineage":["https://openalex.org/I100066346"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"masterThesis"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[{"score":0.6198270320892334,"id":"https://metadata.un.org/sdg/9","display_name":"Industry, innovation and infrastructure"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4410636877","https://openalex.org/W4402670435","https://openalex.org/W4402683864","https://openalex.org/W4404782781","https://openalex.org/W7203611332","https://openalex.org/W4403578570","https://openalex.org/W7143465173","https://openalex.org/W4404792858","https://openalex.org/W4403886154","https://openalex.org/W4414936526"],"abstract_inverted_index":{"Autonomous":[0],"agents":[1,21,39],"that":[2,53],"operate":[3],"within":[4],"graphical":[5],"user":[6,15,28,43,57,78,107],"interfaces":[7],"(GUIs)":[8],"have":[9,184],"a":[10,48,69,94,123,137],"significant":[11],"potential":[12],"to":[13,40,76],"improve":[14],"experience.":[16],"To":[17],"achieve":[18],"this,":[19],"such":[20,109,191,193],"must":[22],"be":[23],"customized":[24],"and":[25,33,88,115,130,155,199,217,250],"proactive.":[26],"Understanding":[27],"intentions":[29,58],"through":[30],"their":[31,60],"interactions":[32],"engagements":[34],"with":[35],"GUIs":[36],"enables":[37],"these":[38],"better":[41],"fulfill":[42],"needs.":[44],"This":[45],"work":[46,232],"introduces":[47],"novel":[49],"LLM-based":[50,173,206],"framework,":[51,145],"Mistral-Intention,":[52],"accurately":[54],"recognizes":[55],"latent":[56],"from":[59,149],"interactions.":[61],"A":[62],"key":[63],"innovation":[64],"is":[65],"the":[66,84,91,100,141,153,179,186,195,201,214,223,236],"integration":[67],"of":[68,93,106,125,140,177,211,238],"sub-goal":[70],"generation":[71],"step,":[72],"using":[73,122],"prompt":[74],"engineering":[75],"decompose":[77],"tasks":[79],"into":[80],"actionable":[81],"steps,":[82],"enhancing":[83],"model's":[85,101],"interpretative":[86],"capabilities":[87],"extendability.":[89],"Furthermore,":[90,169],"incorporation":[92],"keyword":[95],"extraction-based":[96],"loss":[97],"significantly":[98],"refines":[99],"focus":[102],"on":[103,134,152,229],"critical":[104],"information":[105],"actions":[108],"as":[110,194],"typed":[111],"values,":[112],"ensuring":[113],"comprehensive":[114],"relevant":[116],"intention":[117,163],"recognition.":[118],"We":[119,183],"evaluate":[120],"Mistral-Intention":[121],"range":[124],"metrics,":[126],"including":[127],"manual":[128],"metrics":[129],"automatic":[131],"methods":[132],"based":[133],"GPT-4o,":[135],"against":[136],"modified":[138],"version":[139],"state-of-the-art":[142,225],"task":[143],"automation":[144],"namely":[146],"SYNAPSE.":[147],"Results":[148],"extensive":[150],"testing":[151],"MIND2WEB":[154],"MoTIF":[156],"datasets":[157],"highlight":[158],"Mistral-Intention's":[159],"superior":[160],"performance":[161],"in":[162,213,219,227],"recognition":[164],"across":[165],"various":[166],"GUI":[167,203],"environments.":[168],"we":[170],"implement":[171],"an":[172,209],"computer":[174],"agent":[175,207],"capable":[176],"predicting":[178],"user's":[180],"next":[181],"action.":[182],"addressed":[185],"challenges":[187],"faced":[188],"while":[189],"developing":[190,247],"agents,":[192],"limited":[196],"context":[197],"window,":[198],"understanding":[200],"current":[202],"environment.":[204],"Our":[205,231],"exhibits":[208],"improvement":[210],"15.30%":[212],"element":[215],"accuracy":[216],"13.20%":[218],"operation":[220],"F1":[221],"over":[222],"previous":[224],"method":[226],"MindAct":[228],"MIND2WEB.":[230],"not":[233],"only":[234],"pushes":[235],"boundaries":[237],"computational":[239],"HCI":[240],"but":[241],"also":[242],"opens":[243],"new":[244],"pathways":[245],"for":[246],"more":[248],"intuitive":[249],"effective":[251],"user-center":[252],"interaction":[253],"solutions.":[254]},"counts_by_year":[],"updated_date":"2026-08-21T09:56:20.448147","created_date":"2025-10-10T00:00:00"}