{"id":"https://openalex.org/W4393147141","doi":"https://doi.org/10.1609/aaai.v38i16.29799","title":"Improving Knowledge Extraction from LLMs for Task Learning through Agent Analysis","display_name":"Improving Knowledge Extraction from LLMs for Task Learning through Agent Analysis","publication_year":2024,"publication_date":"2024-03-24","ids":{"openalex":"https://openalex.org/W4393147141","doi":"https://doi.org/10.1609/aaai.v38i16.29799"},"language":"en","primary_location":{"id":"doi:10.1609/aaai.v38i16.29799","doi":"https://doi.org/10.1609/aaai.v38i16.29799","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v38i16.29799","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/29799/31382","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://ojs.aaai.org/index.php/AAAI/article/download/29799/31382","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5108741024","display_name":"James R. Kirk","orcid":"https://orcid.org/0000-0002-2459-3643","observed_orcids":["https://orcid.org/0000-0002-2459-3643"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"James R. Kirk","raw_affiliation_strings":["Center for Integrated Cognition"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Center for Integrated Cognition","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011322251","display_name":"Robert E. Wray","orcid":"https://orcid.org/0000-0002-5311-8593","observed_orcids":["https://orcid.org/0000-0002-5311-8593"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Robert E. Wray","raw_affiliation_strings":["Center for Integrated Cognition"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Center for Integrated Cognition","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055620525","display_name":"Peter Lindes","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peter Lindes","raw_affiliation_strings":["Center for Integrated Cognition"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Center for Integrated Cognition","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5061159721","display_name":"John E. Laird","orcid":"https://orcid.org/0000-0001-7446-3241","observed_orcids":["https://orcid.org/0000-0001-7446-3241"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"John E. Laird","raw_affiliation_strings":["Center for Integrated Cognition"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Center for Integrated Cognition","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":4.1227,"has_fulltext":true,"cited_by_count":28,"citation_normalized_percentile":{"value":0.95437674,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":88,"max":99},"biblio":{"volume":"38","issue":"16","first_page":"18390","last_page":"18398"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.6147000193595886,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.6147000193595886,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.10379999876022339,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.062300000339746475,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8029999732971191},{"id":"https://openalex.org/keywords/knowledge-extraction","display_name":"knowledge extraction","score":0.8870000243186951},{"id":"https://openalex.org/keywords/prompt-engineering","display_name":"prompt engineering","score":0.6639999747276306},{"id":"https://openalex.org/keywords/embodied-agents","display_name":"embodied agents","score":0.6970000267028809},{"id":"https://openalex.org/keywords/task-learning","display_name":"task learning","score":0.6349999904632568},{"id":"https://openalex.org/keywords/one-shot-learning","display_name":"one-shot learning","score":0.7429999709129333},{"id":"https://openalex.org/keywords/treatment-response-assessment","display_name":"treatment response assessment","score":0.5569999814033508},{"id":"https://openalex.org/keywords/response-selection","display_name":"response selection","score":0.7889999747276306},{"id":"https://openalex.org/keywords/human-oversight","display_name":"human oversight","score":0.6430000066757202},{"id":"https://openalex.org/keywords/preference-feedback","display_name":"preference feedback","score":0.5839999914169312},{"id":"https://openalex.org/keywords/cognitive-agents","display_name":"cognitive agents","score":0.5950000286102295},{"id":"https://openalex.org/keywords/stars","display_name":"stars","score":0.777999997138977}],"concepts":[{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6441949605941772},{"id":"https://openalex.org/C4725764","wikidata":"https://www.wikidata.org/wiki/Q844704","display_name":"Extraction (chemistry)","level":2,"score":0.4489325284957886},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.11371338367462158},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0934995710849762},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.08970549702644348},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.08383801579475403}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1609/aaai.v38i16.29799","doi":"https://doi.org/10.1609/aaai.v38i16.29799","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v38i16.29799","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/29799/31382","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2306.06770","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2306.06770","pdf_url":"https://arxiv.org/pdf/2306.06770","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:ojs.aaai.org:article/29799","doi":null,"is_oa":false,"landing_page_url":"https://ojs.aaai.org/index.php/AAAI/article/view/29799","pdf_url":null,"source":{"id":"https://openalex.org/S4306400155","display_name":"AAAI Publications (The Association for the Advancement of Artificial Intelligence (AAAI))","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801652627","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/I2801652627"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2159-5399","raw_type":"info:eu-repo/semantics/article"},{"id":"doi:10.48550/arxiv.2306.06770","doi":"https://doi.org/10.48550/arxiv.2306.06770","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2306.06770","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1609/aaai.v38i16.29799","doi":"https://doi.org/10.1609/aaai.v38i16.29799","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v38i16.29799","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/29799/31382","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Quality education","score":0.4156000018119812,"id":"https://metadata.un.org/sdg/4"}],"study_designs":[],"awards":[{"id":"https://openalex.org/G8876996369","display_name":null,"funder_award_id":"N00014","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"}],"funders":[{"id":"https://openalex.org/F4320306078","display_name":"U.S. Department of Defense","ror":"https://ror.org/0447fe631"},{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4393147141.pdf","grobid_xml":"https://content.openalex.org/works/W4393147141.grobid-xml"},"referenced_works_count":22,"referenced_works":["https://openalex.org/W2295722319","https://openalex.org/W2745881055","https://openalex.org/W2966636162","https://openalex.org/W3130319171","https://openalex.org/W3172480024","https://openalex.org/W3192651000","https://openalex.org/W3210904070","https://openalex.org/W4221161695","https://openalex.org/W4224912544","https://openalex.org/W4226112058","https://openalex.org/W4285428875","https://openalex.org/W4287888476","https://openalex.org/W4292958252","https://openalex.org/W4296415305","https://openalex.org/W4312807436","https://openalex.org/W4321011818","https://openalex.org/W4383108457","https://openalex.org/W4386528753","https://openalex.org/W4387835442","https://openalex.org/W4388720459","https://openalex.org/W4402671980","https://openalex.org/W7133210150"],"related_works":["https://openalex.org/W4296415305","https://openalex.org/W7172420227","https://openalex.org/W4391116601","https://openalex.org/W3201464307","https://openalex.org/W4399794854","https://openalex.org/W4407184387","https://openalex.org/W7169865988","https://openalex.org/W4394947983","https://openalex.org/W4401042363","https://openalex.org/W7171737781"],"abstract_inverted_index":{"Large":[0],"language":[1,75,174],"models":[2],"(LLMs)":[3],"offer":[4],"significant":[5],"promise":[6],"as":[7,157],"a":[8,48,129,192],"knowledge":[9,24,38,70],"source":[10],"for":[11,22,33,39],"task":[12,69,139,151],"learning.":[13],"Prompt":[14],"engineering":[15],"has":[16],"been":[17,184],"shown":[18],"to":[19,66,72,86,102,176,191],"be":[20],"effective":[21],"eliciting":[23],"from":[25,133,171],"an":[26,40,64,123,158],"LLM,":[27,135],"but":[28],"alone":[29],"it":[30],"is":[31,85,162],"insufficient":[32],"acquiring":[34],"relevant,":[35],"situationally":[36],"grounded":[37],"embodied":[41],"agent":[42,65,188],"learning":[43,143],"novel":[44],"tasks.":[45],"We":[46,114],"describe":[47,115],"cognitive-agent":[49],"approach,":[50],"STARS,":[51],"that":[52,120,182],"extends":[53],"and":[54,61,79,93,105,118,127],"complements":[55],"prompt":[56],"engineering,":[57],"mitigating":[58],"its":[59,73],"limitations":[60],"thus":[62],"enabling":[63],"acquire":[67],"new":[68],"matched":[71],"native":[74],"capabilities,":[76],"embodiment,":[77],"environment,":[78],"user":[80,145],"preferences.":[81],"The":[82,147],"STARS":[83],"approach":[84,117,148],"increase":[87],"the":[88,99,112,116,134,165,187],"response":[89],"space":[90],"of":[91,131,160,167,179],"LLMs":[92],"deploy":[94],"general":[95],"strategies,":[96],"embedded":[97],"within":[98],"autonomous":[100],"agent,":[101,124],"evaluate,":[103],"repair,":[104],"select":[106],"among":[107],"candidate":[108],"responses":[109,132,181],"produced":[110],"by":[111,125,186],"LLM.":[113],"experiments":[119],"show":[121],"how":[122],"retrieving":[126],"evaluating":[128],"breadth":[130],"can":[136],"achieve":[137],"77-94%":[138],"completion":[140,152],"in":[141],"one-shot":[142],"without":[144],"oversight.":[146],"achieves":[149],"100%":[150],"when":[153],"human":[154],"oversight":[155,168],"(such":[156],"indication":[159],"preference)":[161],"provided.":[163],"Further,":[164],"type":[166],"largely":[169],"shifts":[170],"explicit,":[172],"natural":[173],"instruction":[175],"simple":[177],"confirmation/discomfirmation":[178],"high-quality":[180],"have":[183],"vetted":[185],"before":[189],"presentation":[190],"user.":[193]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":12},{"year":2024,"cited_by_count":9},{"year":2023,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2024-03-26T00:00:00"}