{"id":"https://openalex.org/W4416150558","doi":"https://doi.org/10.48550/arxiv.2502.20175","title":"An Extensive Evaluation of PDDL Capabilities in off-the-shelf LLMs","display_name":"An Extensive Evaluation of PDDL Capabilities in off-the-shelf LLMs","publication_year":2025,"publication_date":"2025-02-27","ids":{"openalex":"https://openalex.org/W4416150558","doi":"https://doi.org/10.48550/arxiv.2502.20175"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2502.20175","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.20175","pdf_url":"https://arxiv.org/pdf/2502.20175","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2502.20175","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5113280447","display_name":"Kaustubh Vyas","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I98677209","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90","country_code":"GB","type":"education","lineage":["https://openalex.org/I98677209"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Vyas, Kaustubh","raw_affiliation_strings":["Huawei Research Ltd. , \u00b6 University of Edinburgh , United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Huawei Research Ltd. , \u00b6 University of Edinburgh , United Kingdom","institution_ids":["https://openalex.org/I98677209"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056690478","display_name":"Damien Graux","orcid":"https://orcid.org/0000-0003-3392-3162","observed_orcids":["https://orcid.org/0000-0003-3392-3162"]},"institutions":[{"id":"https://openalex.org/I98677209","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90","country_code":"GB","type":"education","lineage":["https://openalex.org/I98677209"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Graux, Damien","raw_affiliation_strings":["Huawei Research Ltd. , \u00b6 University of Edinburgh , United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Huawei Research Ltd. , \u00b6 University of Edinburgh , United Kingdom","institution_ids":["https://openalex.org/I98677209"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003791218","display_name":"S\u00e9bastien Montella","orcid":"https://orcid.org/0000-0002-2605-7238","observed_orcids":["https://orcid.org/0000-0002-2605-7238"]},"institutions":[{"id":"https://openalex.org/I98677209","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90","country_code":"GB","type":"education","lineage":["https://openalex.org/I98677209"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Montella, S\u00e9bastien","raw_affiliation_strings":["Huawei Research Ltd. , \u00b6 University of Edinburgh , United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Huawei Research Ltd. , \u00b6 University of Edinburgh , United Kingdom","institution_ids":["https://openalex.org/I98677209"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061611097","display_name":"Pavlos Vougiouklis","orcid":"https://orcid.org/0000-0002-1781-6319","observed_orcids":["https://orcid.org/0000-0002-1781-6319"]},"institutions":[{"id":"https://openalex.org/I98677209","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90","country_code":"GB","type":"education","lineage":["https://openalex.org/I98677209"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Vougiouklis, Pavlos","raw_affiliation_strings":["Huawei Research Ltd. , \u00b6 University of Edinburgh , United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Huawei Research Ltd. , \u00b6 University of Edinburgh , United Kingdom","institution_ids":["https://openalex.org/I98677209"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022971977","display_name":"Ruofei Lai","orcid":"https://orcid.org/0009-0007-0615-8326","observed_orcids":["https://orcid.org/0009-0007-0615-8326"]},"institutions":[{"id":"https://openalex.org/I98677209","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90","country_code":"GB","type":"education","lineage":["https://openalex.org/I98677209"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Lai, Ruofei","raw_affiliation_strings":["Huawei Research Ltd. , \u00b6 University of Edinburgh , United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Huawei Research Ltd. , \u00b6 University of Edinburgh , United Kingdom","institution_ids":["https://openalex.org/I98677209"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Li, Keshuang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I98677209","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90","country_code":"GB","type":"education","lineage":["https://openalex.org/I98677209"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Li, Keshuang","raw_affiliation_strings":["Huawei Research Ltd. , \u00b6 University of Edinburgh , United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Huawei Research Ltd. , \u00b6 University of Edinburgh , United Kingdom","institution_ids":["https://openalex.org/I98677209"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Ren, Yang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I98677209","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90","country_code":"GB","type":"education","lineage":["https://openalex.org/I98677209"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Ren, Yang","raw_affiliation_strings":["Huawei Research Ltd. , \u00b6 University of Edinburgh , United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Huawei Research Ltd. , \u00b6 University of Edinburgh , United Kingdom","institution_ids":["https://openalex.org/I98677209"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5066422711","display_name":"Jeff Z. Pan","orcid":"https://orcid.org/0000-0002-9779-2088","observed_orcids":["https://orcid.org/0000-0002-9779-2088","https://orcid.org/0009-0004-6605-1975","https://orcid.org/0000-0003-0958-8253"]},"institutions":[{"id":"https://openalex.org/I98677209","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90","country_code":"GB","type":"education","lineage":["https://openalex.org/I98677209"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Pan, Jeff Z.","raw_affiliation_strings":["Huawei Research Ltd. , \u00b6 University of Edinburgh , United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Huawei Research Ltd. , \u00b6 University of Edinburgh , United Kingdom","institution_ids":["https://openalex.org/I98677209"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I98677209"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.0010999999940395355,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11010","display_name":"Logic, Reasoning, and Knowledge","score":0.0005000000237487257,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pddl","display_name":"PDDL","score":0.8569999933242798},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9860000014305115},{"id":"https://openalex.org/keywords/formal-planning","display_name":"formal planning","score":0.6629999876022339},{"id":"https://openalex.org/keywords/automated-planning","display_name":"automated planning","score":0.628000020980835},{"id":"https://openalex.org/keywords/zero-shot-learning","display_name":"zero-shot learning","score":0.5389999747276306},{"id":"https://openalex.org/keywords/llm-benchmarking","display_name":"LLM benchmarking","score":0.6190000176429749}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5439000129699707},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.45320001244544983},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.4487999975681305},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.3736000061035156},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.37220001220703125},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.34529998898506165},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.3312000036239624},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.31790000200271606},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.31290000677108765},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3003000020980835},{"id":"https://openalex.org/C105002631","wikidata":"https://www.wikidata.org/wiki/Q4833645","display_name":"Subject-matter expert","level":3,"score":0.2919999957084656},{"id":"https://openalex.org/C55587333","wikidata":"https://www.wikidata.org/wiki/Q1133029","display_name":"Engineering ethics","level":1,"score":0.2809999883174896},{"id":"https://openalex.org/C135257023","wikidata":"https://www.wikidata.org/wiki/Q691358","display_name":"Domain-specific language","level":2,"score":0.27379998564720154},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.25600001215934753},{"id":"https://openalex.org/C157170001","wikidata":"https://www.wikidata.org/wiki/Q4781507","display_name":"Applications of artificial intelligence","level":2,"score":0.2526000142097473}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2502.20175","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.20175","pdf_url":"https://arxiv.org/pdf/2502.20175","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2502.20175","doi":"https://doi.org/10.48550/arxiv.2502.20175","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2502.20175","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2502.20175","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.20175","pdf_url":"https://arxiv.org/pdf/2502.20175","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4416150558.pdf","grobid_xml":"https://content.openalex.org/works/W4416150558.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4414222836","https://openalex.org/W7208782613","https://openalex.org/W7203689856","https://openalex.org/W4412886640","https://openalex.org/W4407309453","https://openalex.org/W4399177056","https://openalex.org/W4393160795","https://openalex.org/W4415797789","https://openalex.org/W4414223502","https://openalex.org/W4415330718"],"abstract_inverted_index":{"In":[0],"recent":[1],"advancements,":[2],"large":[3],"language":[4],"models":[5,56,89],"(LLMs)":[6],"have":[7],"exhibited":[8],"proficiency":[9],"in":[10,44,93,99,117,130],"code":[11],"generation":[12],"and":[13,34,64,79,112,126],"chain-of-thought":[14],"reasoning,":[15],"laying":[16],"the":[17,28,72,110],"groundwork":[18],"for":[19],"tackling":[20],"automatic":[21],"formal":[22,118],"planning":[23,105,119,132],"tasks.":[24],"This":[25],"study":[26],"evaluates":[27],"potential":[29],"of":[30,76,115],"LLMs":[31,116],"to":[32],"understand":[33],"generate":[35],"Planning":[36],"Domain":[37],"Definition":[38],"Language":[39],"(PDDL),":[40],"an":[41,50],"essential":[42],"representation":[43],"artificial":[45],"intelligence":[46],"planning.":[47],"We":[48],"conduct":[49],"extensive":[51],"analysis":[52],"across":[53],"20":[54],"distinct":[55],"spanning":[57],"7":[58],"major":[59],"LLM":[60,74],"families,":[61],"both":[62],"commercial":[63],"open-source.":[65],"Our":[66,83],"comprehensive":[67],"evaluation":[68],"sheds":[69],"light":[70],"on":[71],"zero-shot":[73],"capabilities":[75],"parsing,":[77],"generating,":[78],"reasoning":[80],"with":[81],"PDDL.":[82],"findings":[84],"indicate":[85],"that":[86],"while":[87],"some":[88],"demonstrate":[90],"notable":[91],"effectiveness":[92],"handling":[94],"PDDL,":[95],"others":[96],"pose":[97],"limitations":[98,114],"more":[100],"complex":[101],"scenarios":[102],"requiring":[103],"nuanced":[104],"knowledge.":[106],"These":[107],"results":[108],"highlight":[109],"promise":[111],"current":[113],"tasks,":[120],"offering":[121],"insights":[122],"into":[123],"their":[124],"application":[125],"guiding":[127],"future":[128],"efforts":[129],"AI-driven":[131],"paradigms.":[133]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-11-05T12:25:35.769452","created_date":"2025-10-10T00:00:00"}