{"id":"https://openalex.org/W7164538813","doi":"https://doi.org/10.48550/arxiv.2606.12767","title":"Constructing Evaluation Datasets for Procedural Reasoning: Balancing Naturalness, Grounding, and Multi-Hop Coverage","display_name":"Constructing Evaluation Datasets for Procedural Reasoning: Balancing Naturalness, Grounding, and Multi-Hop Coverage","publication_year":2026,"publication_date":"2026-06-11","ids":{"openalex":"https://openalex.org/W7164538813","doi":"https://doi.org/10.48550/arxiv.2606.12767"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.12767","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.12767","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.12767","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138510916","display_name":"Sarah Elshabrawy","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Elshabrawy, Sarah","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113315309","display_name":"Rahul K. Dass","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dass, Rahul K.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138536890","display_name":"Ashok K. Goel","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Goel, Ashok K.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.521399974822998,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.521399974822998,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.17239999771118164,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.13979999721050262,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/usable","display_name":"USable","score":0.6273000240325928},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5401999950408936},{"id":"https://openalex.org/keywords/procedural-knowledge","display_name":"Procedural knowledge","score":0.4009999930858612},{"id":"https://openalex.org/keywords/structured-prediction","display_name":"Structured prediction","score":0.3440000116825104},{"id":"https://openalex.org/keywords/natural-language-generation","display_name":"Natural language generation","score":0.3427000045776367},{"id":"https://openalex.org/keywords/grounded-theory","display_name":"Grounded theory","score":0.336899995803833}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7105000019073486},{"id":"https://openalex.org/C2780615836","wikidata":"https://www.wikidata.org/wiki/Q2471869","display_name":"USable","level":2,"score":0.6273000240325928},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5401999950408936},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4876999855041504},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4341000020503998},{"id":"https://openalex.org/C124469403","wikidata":"https://www.wikidata.org/wiki/Q1813993","display_name":"Procedural knowledge","level":3,"score":0.4009999930858612},{"id":"https://openalex.org/C22367795","wikidata":"https://www.wikidata.org/wiki/Q7625208","display_name":"Structured prediction","level":2,"score":0.3440000116825104},{"id":"https://openalex.org/C2776187449","wikidata":"https://www.wikidata.org/wiki/Q1513879","display_name":"Natural language generation","level":3,"score":0.3427000045776367},{"id":"https://openalex.org/C156325361","wikidata":"https://www.wikidata.org/wiki/Q1152864","display_name":"Grounded theory","level":3,"score":0.336899995803833},{"id":"https://openalex.org/C132964779","wikidata":"https://www.wikidata.org/wiki/Q2110223","display_name":"Raw data","level":2,"score":0.3296999931335449},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.32899999618530273},{"id":"https://openalex.org/C2776608160","wikidata":"https://www.wikidata.org/wiki/Q4785462","display_name":"Natural (archaeology)","level":2,"score":0.27799999713897705},{"id":"https://openalex.org/C2778348673","wikidata":"https://www.wikidata.org/wiki/Q739302","display_name":"Production (economics)","level":2,"score":0.2768999934196472},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.2750000059604645},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.273499995470047},{"id":"https://openalex.org/C97364631","wikidata":"https://www.wikidata.org/wiki/Q484284","display_name":"Deductive reasoning","level":2,"score":0.25450000166893005}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.12767","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.12767","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.12767","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.12767","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.780096709728241}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Evaluating":[0],"procedural":[1,37,105,160],"reasoning":[2],"in":[3,16,177],"AI-supported":[4,178],"learning":[5],"systems":[6],"requires":[7],"question-answer":[8,114],"datasets":[9,176],"that":[10,60,159],"are":[11,90,98],"both":[12],"learner-like":[13,136],"and":[14,38,57,100,111,128,162],"grounded":[15,126,143],"the":[17,20,93,120],"instructional":[18,109],"knowledge":[19],"system":[21],"is":[22],"expected":[23],"to":[24],"use.":[25],"We":[26,41],"study":[27],"how":[28],"TMK-based":[29],"question":[30],"generation":[31,46,52,59,118,133,147],"strategies":[32],"affect":[33],"dataset":[34],"quality":[35],"for":[36,174],"multi-hop":[39,104,151],"reasoning.":[40,106],"compare":[42],"three":[43],"strategies:":[44],"strict":[45,116],"from":[47,82],"Task-Method-Knowledge":[48],"(TMK)":[49],"models,":[50],"transcript-first":[51],"with":[53,63,124],"post-hoc":[54],"TMK":[55,83,117],"filtering,":[56],"TMK-aware":[58,146],"combines":[61],"transcripts":[62],"structured":[64],"guidance.":[65],"To":[66],"evaluate":[67],"generated":[68,113],"items,":[69,144],"we":[70],"introduce":[71],"a":[72],"grounding":[73],"validation":[74,173],"framework":[75,86],"based":[76],"on":[77],"closed-set":[78],"evidence":[79],"units":[80],"extracted":[81],"models.":[84],"The":[85],"measures":[87],"whether":[88,96,101],"answers":[89],"supported":[91],"by":[92],"underlying":[94],"representation,":[95],"questions":[97,127,137],"self-contained,":[99],"they":[102],"target":[103],"Across":[107],"23":[108],"topics":[110],"690":[112],"pairs,":[115],"achieves":[119],"strongest":[121],"overall":[122],"quality,":[123],"96.5%":[125],"92.6%":[129],"usable":[130],"questions.":[131],"Transcript-first":[132],"produces":[134],"more":[135,139],"but":[138,153],"context-dependent":[140],"or":[141],"weakly":[142],"while":[145],"yields":[148],"high":[149],"raw":[150],"coverage":[152],"lower":[154],"grounding.":[155],"These":[156],"results":[157],"show":[158],"richness":[161],"natural":[163],"phrasing":[164],"do":[165],"not":[166],"guarantee":[167],"representational":[168],"grounding,":[169],"motivating":[170],"explicit":[171],"representation-aware":[172],"evaluation":[175],"learning.":[179]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-13T00:00:00"}