{"id":"https://openalex.org/W7155048810","doi":"https://doi.org/10.48550/arxiv.2604.18177","title":"STaD: Scaffolded Task Design for Identifying Compositional Skill Gaps in LLMs","display_name":"STaD: Scaffolded Task Design for Identifying Compositional Skill Gaps in LLMs","publication_year":2026,"publication_date":"2026-04-20","ids":{"openalex":"https://openalex.org/W7155048810","doi":"https://doi.org/10.48550/arxiv.2604.18177"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.18177","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.18177","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.18177","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5071487399","display_name":"Sungeun An","orcid":"https://orcid.org/0000-0001-5643-9360"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"An, Sungeun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053049612","display_name":"Swanand Kadhe","orcid":"https://orcid.org/0000-0003-2006-1030"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kadhe, Swanand Ravindra","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031485134","display_name":"Shailja Thakur","orcid":"https://orcid.org/0000-0001-9590-5061"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Thakur, Shailja","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051321506","display_name":"Chad DeLuca","orcid":"https://orcid.org/0009-0009-7690-7454"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"DeLuca, Chad","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5102917997","display_name":"Hima Patel","orcid":"https://orcid.org/0000-0002-0031-8428"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Patel, Hima","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.3061000108718872,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.3061000108718872,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.1890999972820282,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.17599999904632568,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7796000242233276},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.7533000111579895},{"id":"https://openalex.org/keywords/aggregate","display_name":"Aggregate (composite)","score":0.48350000381469727},{"id":"https://openalex.org/keywords/strengths-and-weaknesses","display_name":"Strengths and weaknesses","score":0.427700012922287},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.40709999203681946}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7796000242233276},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.7533000111579895},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6151000261306763},{"id":"https://openalex.org/C4679612","wikidata":"https://www.wikidata.org/wiki/Q866298","display_name":"Aggregate (composite)","level":2,"score":0.48350000381469727},{"id":"https://openalex.org/C63882131","wikidata":"https://www.wikidata.org/wiki/Q17122954","display_name":"Strengths and weaknesses","level":2,"score":0.427700012922287},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4156999886035919},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.40709999203681946},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.36239999532699585},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.32089999318122864},{"id":"https://openalex.org/C132758656","wikidata":"https://www.wikidata.org/wiki/Q5307365","display_name":"Dreyfus model of skill acquisition","level":2,"score":0.30799999833106995},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.29409998655319214},{"id":"https://openalex.org/C94966114","wikidata":"https://www.wikidata.org/wiki/Q29256","display_name":"Black box","level":2,"score":0.2856999933719635},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.2822999954223633},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.2605000138282776},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.25380000472068787}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.18177","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.18177","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.18177","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.18177","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.4492553770542145,"display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Benchmarks":[0],"are":[1],"often":[2],"used":[3],"as":[4,93],"a":[5,63,94],"standard":[6],"to":[7,29],"understand":[8],"LLM":[9,92],"capabilities":[10],"in":[11,62,109],"different":[12],"domains.":[13],"However,":[14],"aggregate":[15],"benchmark":[16,49],"scores":[17],"provide":[18],"limited":[19],"insight":[20],"into":[21],"compositional":[22],"skill":[23,86,120],"gaps":[24],"of":[25,48,55,78,102],"LLMs":[26],"and":[27,75,113,118],"how":[28],"improve":[30],"them.":[31],"To":[32],"make":[33],"these":[34],"weaknesses":[35],"visible,":[36],"we":[37],"propose":[38],"Scaffolded":[39],"Task":[40],"Design":[41],"(STaD)":[42],"framework.":[43],"STaD":[44],"generates":[45],"controlled":[46],"variations":[47],"tasks":[50],"based":[51],"on":[52,99],"the":[53,83,91],"concept":[54],"scaffolding,":[56],"which":[57],"introduces":[58],"structured,":[59],"incremental":[60],"support":[61],"step-by-step":[64],"manner.":[65],"Rather":[66],"than":[67],"inspecting":[68],"failures":[69],"individually,":[70],"this":[71],"approach":[72],"enables":[73],"systematic":[74],"scalable":[76],"probing":[77],"model":[79],"behavior":[80],"by":[81],"identifying":[82],"specific":[84],"reasoning":[85,111],"compositions":[87],"they":[88],"lack.":[89],"Treating":[90],"black":[95],"box,":[96],"our":[97],"experiments":[98],"six":[100],"models":[101],"varying":[103],"sizes":[104],"reveal":[105],"multiple":[106],"failure":[107],"points":[108],"three":[110],"benchmarks":[112],"highlight":[114],"each":[115],"model's":[116],"unique":[117],"distinct":[119],"gaps.":[121]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-04-22T00:00:00"}