{"id":"https://openalex.org/W2250817048","doi":"https://doi.org/10.3115/v1/w15-0610","title":"The Impact of Training Data on Automated Short Answer Scoring Performance","display_name":"The Impact of Training Data on Automated Short Answer Scoring Performance","publication_year":2015,"publication_date":"2015-01-01","ids":{"openalex":"https://openalex.org/W2250817048","doi":"https://doi.org/10.3115/v1/w15-0610","mag":"2250817048"},"language":"en","primary_location":{"id":"doi:10.3115/v1/w15-0610","is_oa":true,"landing_page_url":"https://doi.org/10.3115/v1/w15-0610","pdf_url":"https://www.aclweb.org/anthology/W15-0610.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Tenth Workshop on Innovative Use of NLP for Building Educational Applications","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/W15-0610.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5044875578","display_name":"Michael Heilman","orcid":null},"institutions":[{"id":"https://openalex.org/I1341030882","display_name":"Educational Testing Service","ror":"https://ror.org/03b5q4637","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1341030882"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michael Heilman","raw_affiliation_strings":["Educational Testing Service Princeton, NJ, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Educational Testing Service Princeton, NJ, USA","institution_ids":["https://openalex.org/I1341030882"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5015235744","display_name":"Nitin Madnani","orcid":"https://orcid.org/0000-0001-9354-6851"},"institutions":[{"id":"https://openalex.org/I1341030882","display_name":"Educational Testing Service","ror":"https://ror.org/03b5q4637","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1341030882"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nitin Madnani","raw_affiliation_strings":["Educational Testing Service Princeton, NJ, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Educational Testing Service Princeton, NJ, USA","institution_ids":["https://openalex.org/I1341030882"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I1341030882"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":47,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"81","last_page":"85"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.9933000206947327,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7560060620307922},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.6912124156951904},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5525861382484436},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5397511124610901},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.52898108959198},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.52210533618927},{"id":"https://openalex.org/keywords/word","display_name":"Word (group theory)","score":0.4939146637916565},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.43735504150390625},{"id":"https://openalex.org/keywords/data-set","display_name":"Data set","score":0.4365045428276062},{"id":"https://openalex.org/keywords/order","display_name":"Order (exchange)","score":0.4266131818294525},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3405843675136566}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7560060620307922},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.6912124156951904},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5525861382484436},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5397511124610901},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.52898108959198},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.52210533618927},{"id":"https://openalex.org/C90805587","wikidata":"https://www.wikidata.org/wiki/Q10944557","display_name":"Word (group theory)","level":2,"score":0.4939146637916565},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.43735504150390625},{"id":"https://openalex.org/C58489278","wikidata":"https://www.wikidata.org/wiki/Q1172284","display_name":"Data set","level":2,"score":0.4365045428276062},{"id":"https://openalex.org/C182306322","wikidata":"https://www.wikidata.org/wiki/Q1779371","display_name":"Order (exchange)","level":2,"score":0.4266131818294525},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3405843675136566},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C10138342","wikidata":"https://www.wikidata.org/wiki/Q43015","display_name":"Finance","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.3115/v1/w15-0610","is_oa":true,"landing_page_url":"https://doi.org/10.3115/v1/w15-0610","pdf_url":"https://www.aclweb.org/anthology/W15-0610.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Tenth Workshop on Innovative Use of NLP for Building Educational Applications","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.3115/v1/w15-0610","is_oa":true,"landing_page_url":"https://doi.org/10.3115/v1/w15-0610","pdf_url":"https://www.aclweb.org/anthology/W15-0610.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Tenth Workshop on Innovative Use of NLP for Building Educational Applications","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.8899999856948853}],"awards":[{"id":"https://openalex.org/G3371904809","display_name":null,"funder_award_id":"R305F100005","funder_id":"https://openalex.org/F4320306106","funder_display_name":"U.S. Department of Education"},{"id":"https://openalex.org/G6902742591","display_name":"CLASS: Continuous Learning and Automated Scoring in Science","funder_award_id":"1119670","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G963764229","display_name":"Assessing Reading for Understanding: A Theory-based, Developmental Approach","funder_award_id":"R305F100005","funder_id":"https://openalex.org/F4320332210","funder_display_name":"Institute of Education Sciences"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320306106","display_name":"U.S. Department of Education","ror":"https://ror.org/021adze67"},{"id":"https://openalex.org/F4320332210","display_name":"Institute of Education Sciences","ror":"https://ror.org/04et59085"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2250817048.pdf","grobid_xml":"https://content.openalex.org/works/W2250817048.grobid-xml"},"referenced_works_count":10,"referenced_works":["https://openalex.org/W1964357740","https://openalex.org/W1971810500","https://openalex.org/W2020347556","https://openalex.org/W2037789405","https://openalex.org/W2101234009","https://openalex.org/W2115584598","https://openalex.org/W2133436118","https://openalex.org/W2293665254","https://openalex.org/W2335824697","https://openalex.org/W3094565846"],"related_works":["https://openalex.org/W230091440","https://openalex.org/W4394050964","https://openalex.org/W2296205523","https://openalex.org/W2551249631","https://openalex.org/W2786391746","https://openalex.org/W4381430104","https://openalex.org/W2995102745","https://openalex.org/W4226059458","https://openalex.org/W2914559142","https://openalex.org/W1990237101"],"abstract_inverted_index":{"Automatic":[0],"evaluation":[1],"of":[2,18],"written":[3],"responses":[4,64],"to":[5,33,54,65,74,85,102],"content-focused":[6],"assessment":[7],"items":[8],"(automated":[9],"short":[10,99],"answer":[11,100,103],"scoring)":[12],"is":[13,23],"a":[14],"challenging":[15],"educational":[16],"application":[17],"natural":[19],"language":[20],"processing.":[21],"It":[22],"often":[24],"addressed":[25],"using":[26,62],"supervised":[27],"machine":[28],"learning":[29],"by":[30],"estimating":[31],"models":[32],"predict":[34],"human":[35],"scores":[36],"from":[37,68],"detailed":[38],"linguistic":[39],"features":[40],"such":[41,107],"as":[42],"word":[43],"n-grams.":[44],"However,":[45],"training":[46,78,111],"data":[47,112],"(i.e.,":[48],"human-scored":[49],"responses)":[50],"can":[51],"be":[52],"difficult":[53],"acquire.":[55],"In":[56],"this":[57,90],"paper,":[58],"we":[59],"conduct":[60],"experiments":[61],"scored":[63],"44":[66],"prompts":[67],"5":[69],"diverse":[70],"datasets":[71],"in":[72],"order":[73],"better":[75],"understand":[76],"how":[77],"set":[79],"size":[80],"and":[81,95],"other":[82],"factors":[83],"relate":[84],"system":[86],"performance.":[87],"We":[88],"believe":[89],"will":[91],"help":[92],"future":[93],"researchers":[94],"practitioners":[96],"working":[97],"on":[98],"scoring":[101],"practically":[104],"important":[105],"questions":[106],"as,":[108],"\"How":[109],"much":[110],"do":[113],"I":[114],"need?\"":[115],"*":[116]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":7},{"year":2022,"cited_by_count":5},{"year":2021,"cited_by_count":5},{"year":2020,"cited_by_count":4},{"year":2019,"cited_by_count":4},{"year":2018,"cited_by_count":8},{"year":2017,"cited_by_count":4},{"year":2016,"cited_by_count":1},{"year":2015,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}