{"id":"https://openalex.org/W3161616854","doi":"https://doi.org/10.1145/3411764.3445045","title":"Seeing Beyond Expert Blind Spots: Online Learning Design for Scale and Quality","display_name":"Seeing Beyond Expert Blind Spots: Online Learning Design for Scale and Quality","publication_year":2021,"publication_date":"2021-05-06","ids":{"openalex":"https://openalex.org/W3161616854","doi":"https://doi.org/10.1145/3411764.3445045","mag":"3161616854"},"language":"en","primary_location":{"id":"doi:10.1145/3411764.3445045","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3411764.3445045","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3411764.3445045","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102001370","display_name":"Xu Wang","orcid":"https://orcid.org/0000-0003-3509-9146"},"institutions":[{"id":"https://openalex.org/I4210111179","display_name":"Michigan United","ror":"https://ror.org/0291ys696","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210111179"]},{"id":"https://openalex.org/I27837315","display_name":"University of Michigan\u2013Ann Arbor","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Xu Wang","raw_affiliation_strings":["Computer Science and Engineering and University of Michigan, United States"],"affiliations":[{"raw_affiliation_string":"Computer Science and Engineering and University of Michigan, United States","institution_ids":["https://openalex.org/I27837315","https://openalex.org/I4210111179"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089539629","display_name":"Carolyn Penstein Ros\u00e9","orcid":"https://orcid.org/0000-0003-1128-5155"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Carolyn Rose","raw_affiliation_strings":["Human-Computer Interaction Institute Carnegie Mellon University, United States"],"affiliations":[{"raw_affiliation_string":"Human-Computer Interaction Institute Carnegie Mellon University, United States","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5062550465","display_name":"Kenneth R. Koedinger","orcid":"https://orcid.org/0000-0002-5850-4768"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ken Koedinger","raw_affiliation_strings":["Human-Computer Interaction Institute, Carnegie Mellon University, Pittsburgh, Pennsylvania, United States"],"affiliations":[{"raw_affiliation_string":"Human-Computer Interaction Institute, Carnegie Mellon University, Pittsburgh, Pennsylvania, United States","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5102001370"],"corresponding_institution_ids":["https://openalex.org/I27837315","https://openalex.org/I4210111179"],"apc_list":null,"apc_paid":null,"fwci":4.3794,"has_fulltext":false,"cited_by_count":20,"citation_normalized_percentile":{"value":0.94245222,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"14"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10636","display_name":"Innovative Teaching and Learning Methods","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/3204","display_name":"Developmental and Educational Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10636","display_name":"Innovative Teaching and Learning Methods","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/3204","display_name":"Developmental and Educational Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10162","display_name":"Online and Blended Learning","score":0.9926000237464905,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9886999726295471,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.7586976289749146},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7553399801254272},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.6462361216545105},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.5150103569030762},{"id":"https://openalex.org/keywords/instructional-design","display_name":"Instructional design","score":0.47642573714256287},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.4241983890533447},{"id":"https://openalex.org/keywords/multiple-choice","display_name":"Multiple choice","score":0.41875702142715454},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.391001433134079},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.08024734258651733}],"concepts":[{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.7586976289749146},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7553399801254272},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.6462361216545105},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.5150103569030762},{"id":"https://openalex.org/C159456220","wikidata":"https://www.wikidata.org/wiki/Q667334","display_name":"Instructional design","level":2,"score":0.47642573714256287},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.4241983890533447},{"id":"https://openalex.org/C176730311","wikidata":"https://www.wikidata.org/wiki/Q867668","display_name":"Multiple choice","level":3,"score":0.41875702142715454},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.391001433134079},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.08024734258651733},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C126322002","wikidata":"https://www.wikidata.org/wiki/Q11180","display_name":"Internal medicine","level":1,"score":0.0},{"id":"https://openalex.org/C3018023364","wikidata":"https://www.wikidata.org/wiki/Q425265","display_name":"Significant difference","level":2,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3411764.3445045","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3411764.3445045","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3411764.3445045","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3411764.3445045","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.8299999833106995}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":53,"referenced_works":["https://openalex.org/W651343852","https://openalex.org/W1965646104","https://openalex.org/W1973973460","https://openalex.org/W1979290264","https://openalex.org/W1981343066","https://openalex.org/W1989297541","https://openalex.org/W2009845022","https://openalex.org/W2011256040","https://openalex.org/W2013831574","https://openalex.org/W2024253803","https://openalex.org/W2038558034","https://openalex.org/W2038924341","https://openalex.org/W2042400831","https://openalex.org/W2050741681","https://openalex.org/W2068036701","https://openalex.org/W2082954972","https://openalex.org/W2085783790","https://openalex.org/W2092525823","https://openalex.org/W2097279224","https://openalex.org/W2106779500","https://openalex.org/W2108342609","https://openalex.org/W2108765998","https://openalex.org/W2112564774","https://openalex.org/W2122581838","https://openalex.org/W2134907585","https://openalex.org/W2145065862","https://openalex.org/W2153653616","https://openalex.org/W2163569782","https://openalex.org/W2171106563","https://openalex.org/W2220688940","https://openalex.org/W2278680733","https://openalex.org/W2339636024","https://openalex.org/W2397156396","https://openalex.org/W2400788632","https://openalex.org/W2605994771","https://openalex.org/W2609682454","https://openalex.org/W2611910045","https://openalex.org/W2612252595","https://openalex.org/W2612400965","https://openalex.org/W2616951629","https://openalex.org/W2766230222","https://openalex.org/W2796212354","https://openalex.org/W2800912015","https://openalex.org/W2902152098","https://openalex.org/W2910246049","https://openalex.org/W2941035428","https://openalex.org/W2951741171","https://openalex.org/W2966554551","https://openalex.org/W2971755171","https://openalex.org/W2989613245","https://openalex.org/W3029044752","https://openalex.org/W3032136575","https://openalex.org/W3032325918"],"related_works":["https://openalex.org/W2389214306","https://openalex.org/W2965083567","https://openalex.org/W4235240664","https://openalex.org/W1838576100","https://openalex.org/W2757182831","https://openalex.org/W2095886385","https://openalex.org/W2089704382","https://openalex.org/W1983399550","https://openalex.org/W97075385","https://openalex.org/W3132438786"],"abstract_inverted_index":{"Maximizing":[0],"system":[1],"scalability":[2,15],"and":[3,16,46,61,105,145],"quality":[4,17,112],"are":[5,99],"sometimes":[6],"at":[7,21,148],"odds.":[8],"This":[9,132],"work":[10],"provides":[11],"an":[12],"example":[13],"showing":[14],"can":[18],"be":[19],"achieved":[20],"the":[22,41,54,67,129],"same":[23],"time":[24],"in":[25,40,114,139,147],"instructional":[26],"design,":[27],"contrary":[28],"to":[29,49,85,116,127],"what":[30],"instructors":[31,65],"may":[32],"believe":[33],"or":[34,72],"expect.":[35],"We":[36],"situate":[37],"our":[38],"study":[39,120],"education":[42,56],"of":[43,69],"HCI":[44,55,92,151],"methods,":[45],"provide":[47],"suggestions":[48],"improve":[50],"active":[51],"learning":[52,60],"within":[53],"community.":[57],"While":[58],"designing":[59],"assessment":[62,146],"activities,":[63],"many":[64],"face":[66],"choice":[68],"using":[70,107,140],"open-ended":[71,103],"close-ended":[73],"activities.":[74],"Close-ended":[75],"activities":[76],"such":[77],"as":[78],"multiple-choice":[79],"questions":[80],"(MCQs)":[81],"enable":[82],"automated":[83],"feedback":[84],"students.":[86],"However,":[87],"a":[88,95,111],"survey":[89],"with":[90,121],"22":[91],"professors":[93],"revealed":[94],"belief":[96],"that":[97],"MCQs":[98,141],"less":[100],"valuable":[101],"than":[102,137],"questions,":[104],"thus,":[106],"them":[108],"entails":[109],"making":[110],"sacrifice":[113],"order":[115],"achieve":[117],"scalability.":[118],"A":[119],"178":[122],"students":[123],"produced":[124],"no":[125],"evidence":[126],"support":[128],"teacher":[130],"belief.":[131],"paper":[133],"indicates":[134],"more":[135],"promise":[136],"concern":[138],"for":[142],"scalable":[143],"instruction":[144],"least":[149],"some":[150],"domains.":[152]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":5},{"year":2022,"cited_by_count":5},{"year":2021,"cited_by_count":1}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2025-10-10T00:00:00"}