{"id":"https://openalex.org/W3166773976","doi":"https://doi.org/10.1145/3430895.3460141","title":"Domain Experts' Interpretations of Assessment Bias in a Scaled, Online Computer Science Curriculum","display_name":"Domain Experts' Interpretations of Assessment Bias in a Scaled, Online Computer Science Curriculum","publication_year":2021,"publication_date":"2021-06-07","ids":{"openalex":"https://openalex.org/W3166773976","doi":"https://doi.org/10.1145/3430895.3460141","mag":"3166773976"},"language":"en","primary_location":{"id":"doi:10.1145/3430895.3460141","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3430895.3460141","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Eighth ACM Conference on Learning @ Scale","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5032808035","display_name":"Benjamin Xie","orcid":null},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Benjamin Xie","raw_affiliation_strings":["University of Washington, Seatte, WA, USA"],"affiliations":[{"raw_affiliation_string":"University of Washington, Seatte, WA, USA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018037933","display_name":"Matt J. Davidson","orcid":null},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Matt J. Davidson","raw_affiliation_strings":["University of Washington, Seattle, WA, USA"],"affiliations":[{"raw_affiliation_string":"University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034356695","display_name":"Baker Franke","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Baker Franke","raw_affiliation_strings":["Code.org, Seattle, WA, USA"],"affiliations":[{"raw_affiliation_string":"Code.org, Seattle, WA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048819968","display_name":"Emily M. McLeod","orcid":"https://orcid.org/0000-0002-1811-6584"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Emily McLeod","raw_affiliation_strings":["Code.org, Seattle, WA, USA"],"affiliations":[{"raw_affiliation_string":"Code.org, Seattle, WA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100400752","display_name":"Min Li","orcid":"https://orcid.org/0000-0002-7113-2700"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Min Li","raw_affiliation_strings":["University of Washington, Seattle, WA, USA"],"affiliations":[{"raw_affiliation_string":"University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5037160778","display_name":"Amy J. Ko","orcid":"https://orcid.org/0000-0001-7461-4783"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Amy J. Ko","raw_affiliation_strings":["University of Washington, Seattle, WA, USA"],"affiliations":[{"raw_affiliation_string":"University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5032808035"],"corresponding_institution_ids":["https://openalex.org/I201448701"],"apc_list":null,"apc_paid":null,"fwci":2.7313,"has_fulltext":false,"cited_by_count":12,"citation_normalized_percentile":{"value":0.90919313,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"77","last_page":"89"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.979200005531311,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.979200005531311,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12515","display_name":"Gender and Technology in Education","score":0.9779999852180481,"subfield":{"id":"https://openalex.org/subfields/3318","display_name":"Gender Studies"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9732000231742859,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/curriculum","display_name":"Curriculum","score":0.7256277203559875},{"id":"https://openalex.org/keywords/differential-item-functioning","display_name":"Differential item functioning","score":0.690280020236969},{"id":"https://openalex.org/keywords/disadvantaged","display_name":"Disadvantaged","score":0.6734639406204224},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5610854029655457},{"id":"https://openalex.org/keywords/item-response-theory","display_name":"Item response theory","score":0.5054543018341064},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.49163562059402466},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.4366295635700226},{"id":"https://openalex.org/keywords/australian-curriculum","display_name":"Australian Curriculum","score":0.4146692156791687},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.3706715703010559},{"id":"https://openalex.org/keywords/mathematics-education","display_name":"Mathematics education","score":0.359266459941864},{"id":"https://openalex.org/keywords/applied-psychology","display_name":"Applied psychology","score":0.3541908860206604},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.33881402015686035},{"id":"https://openalex.org/keywords/pedagogy","display_name":"Pedagogy","score":0.22749599814414978},{"id":"https://openalex.org/keywords/developmental-psychology","display_name":"Developmental psychology","score":0.18352031707763672},{"id":"https://openalex.org/keywords/psychometrics","display_name":"Psychometrics","score":0.18140476942062378},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.1271207630634308},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.0975981056690216}],"concepts":[{"id":"https://openalex.org/C47177190","wikidata":"https://www.wikidata.org/wiki/Q207137","display_name":"Curriculum","level":2,"score":0.7256277203559875},{"id":"https://openalex.org/C181447626","wikidata":"https://www.wikidata.org/wiki/Q1224359","display_name":"Differential item functioning","level":4,"score":0.690280020236969},{"id":"https://openalex.org/C2780623907","wikidata":"https://www.wikidata.org/wiki/Q106394435","display_name":"Disadvantaged","level":2,"score":0.6734639406204224},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5610854029655457},{"id":"https://openalex.org/C19875794","wikidata":"https://www.wikidata.org/wiki/Q1207340","display_name":"Item response theory","level":3,"score":0.5054543018341064},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.49163562059402466},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.4366295635700226},{"id":"https://openalex.org/C2778567037","wikidata":"https://www.wikidata.org/wiki/Q6972024","display_name":"Australian Curriculum","level":4,"score":0.4146692156791687},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3706715703010559},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.359266459941864},{"id":"https://openalex.org/C75630572","wikidata":"https://www.wikidata.org/wiki/Q538904","display_name":"Applied psychology","level":1,"score":0.3541908860206604},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.33881402015686035},{"id":"https://openalex.org/C19417346","wikidata":"https://www.wikidata.org/wiki/Q7922","display_name":"Pedagogy","level":1,"score":0.22749599814414978},{"id":"https://openalex.org/C138496976","wikidata":"https://www.wikidata.org/wiki/Q175002","display_name":"Developmental psychology","level":1,"score":0.18352031707763672},{"id":"https://openalex.org/C171606756","wikidata":"https://www.wikidata.org/wiki/Q506132","display_name":"Psychometrics","level":2,"score":0.18140476942062378},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.1271207630634308},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0975981056690216},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C151719136","wikidata":"https://www.wikidata.org/wiki/Q3972943","display_name":"Publishing","level":2,"score":0.0},{"id":"https://openalex.org/C165347436","wikidata":"https://www.wikidata.org/wiki/Q1514547","display_name":"Project commissioning","level":3,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3430895.3460141","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3430895.3460141","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Eighth ACM Conference on Learning @ Scale","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.7799999713897705,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G3903816629","display_name":null,"funder_award_id":"1735123, 12566082, 2031265, 1703304, 1539179","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320307764","display_name":"Microsoft","ror":"https://ror.org/00d0nc645"},{"id":"https://openalex.org/F4320307786","display_name":"Adobe Systems","ror":"https://ror.org/059tvcg64"},{"id":"https://openalex.org/F4320309327","display_name":"Google","ror":"https://ror.org/00njsd438"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":78,"referenced_works":["https://openalex.org/W22904917","https://openalex.org/W39076006","https://openalex.org/W73819941","https://openalex.org/W227746905","https://openalex.org/W600396937","https://openalex.org/W603915373","https://openalex.org/W615644768","https://openalex.org/W629063538","https://openalex.org/W1502678715","https://openalex.org/W1562297848","https://openalex.org/W1586414640","https://openalex.org/W1594501748","https://openalex.org/W1720694869","https://openalex.org/W1814344639","https://openalex.org/W1815596065","https://openalex.org/W1913957972","https://openalex.org/W1919216659","https://openalex.org/W1949744349","https://openalex.org/W1963760706","https://openalex.org/W1969162744","https://openalex.org/W1986031600","https://openalex.org/W1987634243","https://openalex.org/W1988057844","https://openalex.org/W1990672191","https://openalex.org/W2000130928","https://openalex.org/W2029221441","https://openalex.org/W2048149652","https://openalex.org/W2050239379","https://openalex.org/W2053070609","https://openalex.org/W2067867407","https://openalex.org/W2070629719","https://openalex.org/W2074257409","https://openalex.org/W2077712102","https://openalex.org/W2089262464","https://openalex.org/W2096160735","https://openalex.org/W2106005988","https://openalex.org/W2108104166","https://openalex.org/W2110065044","https://openalex.org/W2117411972","https://openalex.org/W2138587974","https://openalex.org/W2188960590","https://openalex.org/W2484197944","https://openalex.org/W2573932743","https://openalex.org/W2620837250","https://openalex.org/W2765549052","https://openalex.org/W2783198183","https://openalex.org/W2784935351","https://openalex.org/W2799034216","https://openalex.org/W2800160966","https://openalex.org/W2809049841","https://openalex.org/W2810769058","https://openalex.org/W2887900586","https://openalex.org/W2914554564","https://openalex.org/W2916082316","https://openalex.org/W2917159054","https://openalex.org/W2922154011","https://openalex.org/W2958750980","https://openalex.org/W2963896862","https://openalex.org/W2966399840","https://openalex.org/W2998674842","https://openalex.org/W3006952330","https://openalex.org/W3018063079","https://openalex.org/W3035884323","https://openalex.org/W3046417578","https://openalex.org/W3046515610","https://openalex.org/W3084611909","https://openalex.org/W3112698404","https://openalex.org/W3113961228","https://openalex.org/W3127631298","https://openalex.org/W3133899330","https://openalex.org/W3135573294","https://openalex.org/W3157548865","https://openalex.org/W3177367885","https://openalex.org/W4211086681","https://openalex.org/W4239260833","https://openalex.org/W4247528512","https://openalex.org/W4251755993","https://openalex.org/W6600457792"],"related_works":["https://openalex.org/W1968852390","https://openalex.org/W2186771857","https://openalex.org/W2370894755","https://openalex.org/W4234875053","https://openalex.org/W1957521962","https://openalex.org/W2799539349","https://openalex.org/W2030595022","https://openalex.org/W2918541347","https://openalex.org/W2088715419","https://openalex.org/W2000333604"],"abstract_inverted_index":{"Understanding":[0],"inequity":[1],"at":[2],"scale":[3],"is":[4],"necessary":[5],"for":[6,194],"designing":[7],"equitable":[8,217],"online":[9,86],"learning":[10,218],"experiences,":[11],"but":[12],"also":[13,138],"difficult.":[14],"Statistical":[15],"techniques":[16],"like":[17],"differential":[18],"item":[19,39,176,183],"functioning":[20],"(DIF)":[21],"can":[22],"help":[23],"identify":[24,54,100],"whether":[25,38],"items/questions":[26],"in":[27,106,200],"an":[28,180],"assessment":[29,107,208],"exhibit":[30],"potential":[31],"bias":[32,177,209],"by":[33,102],"disadvantaging":[34],"certain":[35],"groups":[36],"(e.g.":[37,109],"disadvantages":[40],"woman":[41],"vs":[42],"man":[43],"of":[44,182,204],"equivalent":[45],"knowledge).":[46],"While":[47],"testing":[48],"companies":[49],"typically":[50],"use":[51,67],"DIF":[52,68,101],"to":[53,56,69,73,77,99,129,155,179,210,214],"items":[55,108,143],"remove,":[57],"we":[58,92,117],"explored":[59],"how":[60,72],"domain-experts":[61],"such":[62],"as":[63,125,133],"curriculum":[64,169],"designers":[65,170],"could":[66],"better":[70,78],"understand":[71],"design":[74,215],"instructional":[75],"materials":[76],"serve":[79],"students":[80,98,122,130,153],"from":[81,96],"diverse":[82],"groups.":[83],"Using":[84],"Code.org's":[85],"Computer":[87],"Science":[88],"Discoveries":[89],"(CSD)":[90],"curriculum,":[91,190],"analyzed":[93],"139,097":[94],"responses":[95],"19,617":[97],"gender":[103],"and":[104,165,171,185,191],"race":[105],"multiple":[110],"choice":[111],"questions).":[112],"Of":[113],"the":[114,188,201],"17":[115],"items,":[116],"identified":[118,139],"six":[119],"that":[120,140,173],"disadvantaged":[121,144],"who":[123,131],"reported":[124,132],"female":[126],"when":[127],"compared":[128,154],"non-binary":[134],"or":[135],"male.":[136],"We":[137,160,196],"most":[141],"(13)":[142],"AHNP":[145],"(African/Black,":[146],"Hispanic/Latinx,":[147],"Native":[148],"American/Alaskan":[149],"Native,":[150],"Pacific":[151],"Islander)":[152],"WA":[156],"(white,":[157],"Asian)":[158],"students.":[159],"then":[161],"conducted":[162],"a":[163],"workshop":[164],"interviews":[166],"with":[167],"seven":[168],"found":[172],"they":[174],"interpreted":[175,197],"relative":[178],"intersection":[181],"features":[184],"student":[186],"identity,":[187],"broader":[189,202],"differing":[192],"uses":[193],"assessments.":[195],"these":[198],"findings":[199],"context":[203],"using":[205],"data":[206],"on":[207],"inform":[211],"domain-experts'":[212],"efforts":[213],"more":[216],"experiences.":[219]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":4},{"year":2022,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}