{"id":"https://openalex.org/W4307309337","doi":"https://doi.org/10.52041/iase.icots11.t3c3","title":"Foundations for AI-Assisted Formative Assessment Feedback for Short-Answer Tasks in Large-Enrollment Classes","display_name":"Foundations for AI-Assisted Formative Assessment Feedback for Short-Answer Tasks in Large-Enrollment Classes","publication_year":2022,"publication_date":"2022-10-26","ids":{"openalex":"https://openalex.org/W4307309337","doi":"https://doi.org/10.52041/iase.icots11.t3c3"},"language":"en","primary_location":{"id":"doi:10.52041/iase.icots11.t3c3","doi":"https://doi.org/10.52041/iase.icots11.t3c3","is_oa":true,"landing_page_url":"http://doi.org/10.52041/iase.icots11.t3c3","pdf_url":"https://doi.org/10.52041/iase.icots11.t3c3","source":{"id":"https://openalex.org/S4363608843","display_name":"Bridging the Gap: Empowering and Educating Today\u2019s Learners in Statistics. Proceedings of the Eleventh International Conference on Teaching Statistics","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Bridging the Gap: Empowering and Educating Today\u2019s Learners in Statistics. Proceedings of the Eleventh International Conference on Teaching Statistics","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://doi.org/10.52041/iase.icots11.t3c3","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5109181543","display_name":"Susan Lloyd","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I130769515","display_name":"Pennsylvania State University","ror":"https://ror.org/04p491231","country_code":"US","type":"education","lineage":["https://openalex.org/I130769515"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Susan Lloyd","raw_affiliation_strings":["The Pennsylvania State University , USA","The Pennsylvania State University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Pennsylvania State University , USA","institution_ids":["https://openalex.org/I130769515"]},{"raw_affiliation_string":"The Pennsylvania State University","institution_ids":["https://openalex.org/I130769515"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Matthew Beckman","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I130769515","display_name":"Pennsylvania State University","ror":"https://ror.org/04p491231","country_code":"US","type":"education","lineage":["https://openalex.org/I130769515"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Matthew Beckman","raw_affiliation_strings":["The Pennsylvania State University , USA","The Pennsylvania State University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Pennsylvania State University , USA","institution_ids":["https://openalex.org/I130769515"]},{"raw_affiliation_string":"The Pennsylvania State University","institution_ids":["https://openalex.org/I130769515"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057360121","display_name":"Dennis K. Pearl","orcid":"https://orcid.org/0000-0003-1981-1826","observed_orcids":["https://orcid.org/0000-0003-1981-1826"]},"institutions":[{"id":"https://openalex.org/I130769515","display_name":"Pennsylvania State University","ror":"https://ror.org/04p491231","country_code":"US","type":"education","lineage":["https://openalex.org/I130769515"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dennis Pearl","raw_affiliation_strings":["The Pennsylvania State University , USA","The Pennsylvania State University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Pennsylvania State University , USA","institution_ids":["https://openalex.org/I130769515"]},{"raw_affiliation_string":"The Pennsylvania State University","institution_ids":["https://openalex.org/I130769515"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039621382","display_name":"Rebecca J. Passonneau","orcid":"https://orcid.org/0000-0001-8626-811X","observed_orcids":["https://orcid.org/0000-0001-8626-811X"]},"institutions":[{"id":"https://openalex.org/I130769515","display_name":"Pennsylvania State University","ror":"https://ror.org/04p491231","country_code":"US","type":"education","lineage":["https://openalex.org/I130769515"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rebecca Passonneau","raw_affiliation_strings":["The Pennsylvania State University , USA","The Pennsylvania State University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Pennsylvania State University , USA","institution_ids":["https://openalex.org/I130769515"]},{"raw_affiliation_string":"The Pennsylvania State University","institution_ids":["https://openalex.org/I130769515"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100376717","display_name":"Zhaohui Li","orcid":"https://orcid.org/0000-0003-4009-4374","observed_orcids":["https://orcid.org/0000-0003-4009-4374"]},"institutions":[{"id":"https://openalex.org/I130769515","display_name":"Pennsylvania State University","ror":"https://ror.org/04p491231","country_code":"US","type":"education","lineage":["https://openalex.org/I130769515"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhaohui Li","raw_affiliation_strings":["The Pennsylvania State University , USA","The Pennsylvania State University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Pennsylvania State University , USA","institution_ids":["https://openalex.org/I130769515"]},{"raw_affiliation_string":"The Pennsylvania State University","institution_ids":["https://openalex.org/I130769515"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100776738","display_name":"Zekun Wang","orcid":"https://orcid.org/0000-0001-8631-7022","observed_orcids":["https://orcid.org/0000-0001-8631-7022"]},"institutions":[{"id":"https://openalex.org/I130769515","display_name":"Pennsylvania State University","ror":"https://ror.org/04p491231","country_code":"US","type":"education","lineage":["https://openalex.org/I130769515"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zekun Wang","raw_affiliation_strings":["The Pennsylvania State University , USA","The Pennsylvania State University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Pennsylvania State University , USA","institution_ids":["https://openalex.org/I130769515"]},{"raw_affiliation_string":"The Pennsylvania State University","institution_ids":["https://openalex.org/I130769515"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I130769515"],"apc_list":null,"apc_paid":null,"fwci":3.4074,"has_fulltext":true,"cited_by_count":8,"citation_normalized_percentile":{"value":0.90470867,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":88,"max":98},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.9018999934196472,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.9018999934196472,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T14025","display_name":"Educational Technology and Assessment","score":0.036400001496076584,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.03150000050663948,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/formative-assessment","display_name":"formative assessment","score":0.8299999833106995},{"id":"https://openalex.org/keywords/automated-essay-scoring","display_name":"automated essay scoring","score":0.5460000038146973},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"natural language processing","score":0.8450000286102295},{"id":"https://openalex.org/keywords/large-classes","display_name":"large classes","score":0.6380000114440918},{"id":"https://openalex.org/keywords/interobserver-agreement","display_name":"interobserver agreement","score":0.8889999985694885},{"id":"https://openalex.org/keywords/scalable-assessment","display_name":"scalable assessment","score":0.6869999766349792},{"id":"https://openalex.org/keywords/statistics-education","display_name":"statistics education","score":0.7979999780654907}],"concepts":[{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.8976833820343018},{"id":"https://openalex.org/C42525527","wikidata":"https://www.wikidata.org/wiki/Q1209955","display_name":"Formative assessment","level":2,"score":0.8966245651245117},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6303056478500366},{"id":"https://openalex.org/C2778724333","wikidata":"https://www.wikidata.org/wiki/Q14401","display_name":"Kappa","level":2,"score":0.6022608280181885},{"id":"https://openalex.org/C2780876879","wikidata":"https://www.wikidata.org/wiki/Q3054749","display_name":"Meaning (existential)","level":2,"score":0.46234551072120667},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.45609015226364136},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4558084011077881},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.4410306215286255},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4111679196357727},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3616846203804016},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.32643330097198486},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.14754906296730042},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C542102704","wikidata":"https://www.wikidata.org/wiki/Q183257","display_name":"Psychotherapist","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.52041/iase.icots11.t3c3","doi":"https://doi.org/10.52041/iase.icots11.t3c3","is_oa":true,"landing_page_url":"http://doi.org/10.52041/iase.icots11.t3c3","pdf_url":"https://doi.org/10.52041/iase.icots11.t3c3","source":{"id":"https://openalex.org/S4363608843","display_name":"Bridging the Gap: Empowering and Educating Today\u2019s Learners in Statistics. Proceedings of the Eleventh International Conference on Teaching Statistics","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Bridging the Gap: Empowering and Educating Today\u2019s Learners in Statistics. Proceedings of the Eleventh International Conference on Teaching Statistics","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2205.02829","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2205.02829","pdf_url":"https://arxiv.org/pdf/2205.02829","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2205.02829","doi":"https://doi.org/10.48550/arxiv.2205.02829","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2205.02829","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article-journal"}],"best_oa_location":{"id":"doi:10.52041/iase.icots11.t3c3","doi":"https://doi.org/10.52041/iase.icots11.t3c3","is_oa":true,"landing_page_url":"http://doi.org/10.52041/iase.icots11.t3c3","pdf_url":"https://doi.org/10.52041/iase.icots11.t3c3","source":{"id":"https://openalex.org/S4363608843","display_name":"Bridging the Gap: Empowering and Educating Today\u2019s Learners in Statistics. Proceedings of the Eleventh International Conference on Teaching Statistics","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Bridging the Gap: Empowering and Educating Today\u2019s Learners in Statistics. Proceedings of the Eleventh International Conference on Teaching Statistics","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.975600004196167,"display_name":"Quality education"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4307309337.pdf","grobid_xml":"https://content.openalex.org/works/W4307309337.grobid-xml"},"referenced_works_count":19,"referenced_works":["https://openalex.org/W1444168786","https://openalex.org/W1580749063","https://openalex.org/W1864276097","https://openalex.org/W1973517668","https://openalex.org/W1996290308","https://openalex.org/W2001402876","https://openalex.org/W2076416532","https://openalex.org/W2088987187","https://openalex.org/W2135875128","https://openalex.org/W2183293350","https://openalex.org/W2226404023","https://openalex.org/W2287330439","https://openalex.org/W2766485329","https://openalex.org/W2990732585","https://openalex.org/W3011616684","https://openalex.org/W3046630816","https://openalex.org/W3154168950","https://openalex.org/W3214533270","https://openalex.org/W3217611625"],"related_works":["https://openalex.org/W7214228150","https://openalex.org/W4413323296","https://openalex.org/W2244033128","https://openalex.org/W4417477673","https://openalex.org/W7204200228","https://openalex.org/W7111859580","https://openalex.org/W2590462553","https://openalex.org/W7214079160","https://openalex.org/W3173436404","https://openalex.org/W7095751736"],"abstract_inverted_index":{"Research":[0],"suggests":[1],"\u201cwrite-to-learn\u201d":[2],"tasks":[3,32],"improve":[4],"learning":[5],"outcomes,":[6],"yet":[7],"constructed-response":[8],"methods":[9],"of":[10,102],"formative":[11,118],"assessment":[12],"become":[13],"unwieldy":[14],"with":[15],"large":[16],"class":[17],"sizes.":[18],"This":[19],"study":[20,97],"evaluates":[21],"natural":[22],"language":[23],"processing":[24],"algorithms":[25],"to":[26,108,111],"assist":[27],"this":[28],"aim.":[29],"Six":[30],"short-answer":[31],"completed":[33],"by":[34,39],"1,935":[35],"students":[36],"were":[37],"scored":[38,83],"several":[40],"human":[41],"raters":[42],"using":[43,55],"a":[44,114],"detailed":[45],"rubric":[46],"and":[47,66],"an":[48],"algorithm.":[49],"Results":[50],"indicate":[51],"substantial":[52],"inter-rater":[53],"agreement":[54,75],"quadratic":[56],"weighted":[57],"kappa":[58],"for":[59,78,116],"rater":[60,80,94],"pairs":[61],"(each":[62],"QWK":[63],">":[64],"0.74)":[65],"group":[67],"consensus":[68],"(Fleiss\u2019":[69],"Kappa":[70],"=":[71,90],"0.68).":[72],"Additionally,":[73],"intra-rater":[74],"was":[76],"estimated":[77],"one":[79],"who":[81],"had":[82],"178":[84],"responses":[85],"seven":[86],"years":[87],"prior":[88],"(QWK":[89],"0.88).":[91],"With":[92],"compelling":[93],"agreement,":[95],"the":[96],"then":[98],"pilots":[99],"cluster":[100],"analysis":[101],"response":[103],"text":[104],"toward":[105],"enabling":[106],"instructors":[107],"ascribe":[109],"meaning":[110],"clusters":[112],"as":[113],"means":[115],"scalable":[117],"assessment.":[119]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2022-10-31T00:00:00"}