{"id":"https://openalex.org/W7162330764","doi":"https://doi.org/10.48550/arxiv.2605.23657","title":"OpenSkillEval: Automatically Auditing the Open Skill Ecosystem for LLM Agents","display_name":"OpenSkillEval: Automatically Auditing the Open Skill Ecosystem for LLM Agents","publication_year":2026,"publication_date":"2026-05-22","ids":{"openalex":"https://openalex.org/W7162330764","doi":"https://doi.org/10.48550/arxiv.2605.23657"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.23657","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.23657","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.23657","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136935490","display_name":"Jiahao Ying","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ying, Jiahao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127818013","display_name":"Boxian Ai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ai, Boxian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136919341","display_name":"Wei Tang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tang, Wei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136964817","display_name":"Siyuan Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Siyuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136987937","display_name":"Yixin Cao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cao, Yixin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.1120000034570694,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.1120000034570694,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0843999981880188,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.07460000365972519,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6935999989509583},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.6722999811172485},{"id":"https://openalex.org/keywords/audit","display_name":"Audit","score":0.6208000183105469},{"id":"https://openalex.org/keywords/knowledge-base","display_name":"Knowledge base","score":0.47209998965263367},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.4602999985218048},{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.43380001187324524}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7476999759674072},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6935999989509583},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.6722999811172485},{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.6208000183105469},{"id":"https://openalex.org/C4554734","wikidata":"https://www.wikidata.org/wiki/Q593744","display_name":"Knowledge base","level":2,"score":0.47209998965263367},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.4602999985218048},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.43380001187324524},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.41620001196861267},{"id":"https://openalex.org/C2776207758","wikidata":"https://www.wikidata.org/wiki/Q5303302","display_name":"Downstream (manufacturing)","level":2,"score":0.3443000018596649},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.3400000035762787},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3384999930858612},{"id":"https://openalex.org/C2777601897","wikidata":"https://www.wikidata.org/wiki/Q3409113","display_name":"Presentation (obstetrics)","level":2,"score":0.328900009393692},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.323199987411499},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.31290000677108765},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.28760001063346863},{"id":"https://openalex.org/C96711827","wikidata":"https://www.wikidata.org/wiki/Q17012245","display_name":"Entity linking","level":3,"score":0.2806999981403351},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.2651999890804291},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.26420000195503235},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.2572000026702881}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.23657","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.23657","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.23657","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.23657","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.6994507908821106,"display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Skills,":[0],"i.e.,":[1],"structured":[2],"workflow":[3],"instructions":[4],"distilled":[5],"for":[6,17,70,120,198,214],"large":[7],"language":[8],"models":[9,38,146],"(LLMs),":[10],"are":[11,222],"becoming":[12],"an":[13,66],"increasingly":[14],"important":[15],"mechanism":[16],"improving":[18],"agent":[19,40,73,148,177],"performance":[20],"on":[21,82,170,224],"real-world":[22,93],"downstream":[23,99],"tasks.":[24],"However,":[25],"as":[26],"the":[27,76,163,172,176,196,207,225],"open-source":[28,137],"skill":[29,48,154,160,166],"ecosystem":[30],"rapidly":[31],"expands,":[32],"it":[33],"remains":[34],"unclear":[35],"how":[36,45,51],"different":[37],"and":[39,50,75,110,116,135,147,175,179,202,210,219],"frameworks":[41],"interact":[42],"with":[43],"skills,":[44,138],"to":[46],"evaluate":[47],"quality,":[49],"users":[52],"should":[53],"select":[54],"skills":[55,77,119,184,213],"under":[56,123],"practical":[57,204],"cost-performance":[58],"trade-offs.":[59],"In":[60],"this":[61],"paper,":[62],"we":[63,139],"present":[64],"\\textsc{OpenSkillEval},":[65],"automatic":[67],"evaluation":[68,143,201],"framework":[69],"both":[71,171],"skill-augmented":[72],"systems":[74],"themselves.":[78],"Instead":[79],"of":[80,98,144,165,212],"relying":[81],"static":[83],"benchmarks,":[84],"\\textsc{OpenSkillEval}":[85],"automatically":[86],"constructs":[87],"realistic":[88],"task":[89,125,133],"instances":[90,134],"from":[91],"evolving":[92],"artifacts":[94],"across":[95],"five":[96],"categories":[97],"applications:":[100],"presentation":[101],"generation,":[102,107],"front-end":[103],"web":[104],"design,":[105,208],"poster":[106],"data":[108],"visualization,":[109],"report":[111],"generation.":[112],"It":[113],"further":[114],"collects":[115],"organizes":[117],"community-contributed":[118],"controlled":[121],"comparison":[122],"unified":[124],"settings.":[126],"Using":[127],"more":[128],"than":[129],"600":[130],"dynamically":[131],"generated":[132],"30":[136],"conduct":[140],"a":[141],"systematic":[142],"state-of-the-art":[145],"frameworks.":[149],"Our":[150],"results":[151],"show":[152],"that":[153,162,180],"availability":[155],"does":[156],"not":[157,186],"guarantee":[158],"effective":[159],"usage,":[161],"benefit":[164],"augmentation":[167],"depends":[168],"strongly":[169],"underlying":[173],"model":[174],"framework,":[178],"many":[181],"publicly":[182],"popular":[183],"do":[185],"consistently":[187],"outperform":[188],"base":[189],"agents":[190],"without":[191],"skills.":[192],"These":[193],"findings":[194],"highlight":[195],"need":[197],"dynamic,":[199],"task-grounded":[200],"provide":[203],"insights":[205],"into":[206],"selection,":[209],"deployment":[211],"LLM":[215],"agents.":[216],"Additional":[217],"cases":[218],"benchmark":[220],"resources":[221],"available":[223],"project":[226],"website:":[227],"https://yingjiahao14.github.io/OpenSkillEval-Web/.":[228]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-26T00:00:00"}