{"id":"https://openalex.org/W3157782429","doi":"https://doi.org/10.1145/3449193","title":"Can Crowds Customize Instructional Materials with Minimal Expert Guidance?","display_name":"Can Crowds Customize Instructional Materials with Minimal Expert Guidance?","publication_year":2021,"publication_date":"2021-04-13","ids":{"openalex":"https://openalex.org/W3157782429","doi":"https://doi.org/10.1145/3449193","mag":"3157782429"},"language":"en","primary_location":{"id":"doi:10.1145/3449193","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3449193","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3449193","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3449193","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5024138893","display_name":"Kexin Yang","orcid":"https://orcid.org/0000-0002-8132-0166"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Kexin Bella Yang","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049383644","display_name":"Tomohiro Nagashima","orcid":"https://orcid.org/0000-0003-2489-5016"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tomohiro Nagashima","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020485512","display_name":"Junhui Yao","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Junhui Yao","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069476228","display_name":"Joseph Jay Williams","orcid":"https://orcid.org/0000-0002-9122-5242"},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Joseph Jay Williams","raw_affiliation_strings":["University of Toronto, Toronto, ON, Canada"],"affiliations":[{"raw_affiliation_string":"University of Toronto, Toronto, ON, Canada","institution_ids":["https://openalex.org/I185261750"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022664382","display_name":"Kenneth Holstein","orcid":"https://orcid.org/0000-0001-6730-922X"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kenneth Holstein","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5047522207","display_name":"Vincent Aleven","orcid":"https://orcid.org/0000-0002-1581-6657"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Vincent Aleven","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5024138893"],"corresponding_institution_ids":["https://openalex.org/I74973139"],"apc_list":null,"apc_paid":null,"fwci":1.9473,"has_fulltext":true,"cited_by_count":9,"citation_normalized_percentile":{"value":0.87906081,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":97},"biblio":{"volume":"5","issue":"CSCW1","first_page":"1","last_page":"24"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9884999990463257,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.9754999876022339,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9403173327445984},{"id":"https://openalex.org/keywords/crowds","display_name":"Crowds","score":0.8003453612327576},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6703541278839111},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.668605387210846},{"id":"https://openalex.org/keywords/personalization","display_name":"Personalization","score":0.6359817981719971},{"id":"https://openalex.org/keywords/tutor","display_name":"TUTOR","score":0.5770902037620544},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.4409751296043396},{"id":"https://openalex.org/keywords/mathematics-education","display_name":"Mathematics education","score":0.37408190965652466},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.36921626329421997},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.3675418198108673},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.34701764583587646},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.15088418126106262},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.13662931323051453}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9403173327445984},{"id":"https://openalex.org/C2777852691","wikidata":"https://www.wikidata.org/wiki/Q13430821","display_name":"Crowds","level":2,"score":0.8003453612327576},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6703541278839111},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.668605387210846},{"id":"https://openalex.org/C183003079","wikidata":"https://www.wikidata.org/wiki/Q1000371","display_name":"Personalization","level":2,"score":0.6359817981719971},{"id":"https://openalex.org/C2778371403","wikidata":"https://www.wikidata.org/wiki/Q7672049","display_name":"TUTOR","level":2,"score":0.5770902037620544},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4409751296043396},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.37408190965652466},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.36921626329421997},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.3675418198108673},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.34701764583587646},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.15088418126106262},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.13662931323051453},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3449193","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3449193","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3449193","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3449193","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3449193","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3449193","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.699999988079071}],"awards":[{"id":"https://openalex.org/G1980104386","display_name":null,"funder_award_id":"#1822861","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G2637195115","display_name":null,"funder_award_id":"4-18-1-","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"},{"id":"https://openalex.org/G2811237814","display_name":null,"funder_award_id":"Grant","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"},{"id":"https://openalex.org/G4039941258","display_name":null,"funder_award_id":"1822861","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5117620571","display_name":null,"funder_award_id":"N00014-18-1-2755","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"},{"id":"https://openalex.org/G848032724","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8876996369","display_name":null,"funder_award_id":"N00014","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"},{"id":"https://openalex.org/G940897683","display_name":null,"funder_award_id":"#N00014-18-1-2755","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3157782429.pdf","grobid_xml":"https://content.openalex.org/works/W3157782429.grobid-xml"},"referenced_works_count":74,"referenced_works":["https://openalex.org/W90337360","https://openalex.org/W110184136","https://openalex.org/W133284495","https://openalex.org/W182096324","https://openalex.org/W303020954","https://openalex.org/W410845123","https://openalex.org/W1518435216","https://openalex.org/W1570348318","https://openalex.org/W1599803565","https://openalex.org/W1925613010","https://openalex.org/W1971228271","https://openalex.org/W1973490896","https://openalex.org/W1985824294","https://openalex.org/W1990646004","https://openalex.org/W2014100104","https://openalex.org/W2015463865","https://openalex.org/W2018300363","https://openalex.org/W2028953510","https://openalex.org/W2041668724","https://openalex.org/W2058179030","https://openalex.org/W2067652191","https://openalex.org/W2074275752","https://openalex.org/W2075433557","https://openalex.org/W2091366170","https://openalex.org/W2105230945","https://openalex.org/W2105549794","https://openalex.org/W2106519279","https://openalex.org/W2111009050","https://openalex.org/W2116915306","https://openalex.org/W2117470435","https://openalex.org/W2117655204","https://openalex.org/W2133485007","https://openalex.org/W2134964681","https://openalex.org/W2135995390","https://openalex.org/W2139891935","https://openalex.org/W2148479118","https://openalex.org/W2151401338","https://openalex.org/W2151785234","https://openalex.org/W2152378008","https://openalex.org/W2154041216","https://openalex.org/W2168144930","https://openalex.org/W2168765606","https://openalex.org/W2234716128","https://openalex.org/W2237600160","https://openalex.org/W2273188821","https://openalex.org/W2276569499","https://openalex.org/W2288244345","https://openalex.org/W2288486118","https://openalex.org/W2290102723","https://openalex.org/W2296735967","https://openalex.org/W2313601806","https://openalex.org/W2336395599","https://openalex.org/W2400269077","https://openalex.org/W2461712330","https://openalex.org/W2559122630","https://openalex.org/W2577008892","https://openalex.org/W2592229992","https://openalex.org/W2599032673","https://openalex.org/W2604446244","https://openalex.org/W2734400156","https://openalex.org/W2796371790","https://openalex.org/W2808967136","https://openalex.org/W2900340839","https://openalex.org/W2952696298","https://openalex.org/W2963597902","https://openalex.org/W2966032592","https://openalex.org/W3018065846","https://openalex.org/W3127529611","https://openalex.org/W3134814310","https://openalex.org/W3164105584","https://openalex.org/W4230939871","https://openalex.org/W4243863958","https://openalex.org/W4243916257","https://openalex.org/W6679756477"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W1503094549","https://openalex.org/W4384486036","https://openalex.org/W135177976","https://openalex.org/W2337920774","https://openalex.org/W4318823662","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W3207526114","https://openalex.org/W4286908577"],"abstract_inverted_index":{"AI-based":[0,89],"educational":[1],"technologies":[2],"may":[3,173],"be":[4,33,56],"most":[5],"welcome":[6],"in":[7,63,149,175,184,261,273],"classrooms":[8],"when":[9],"they":[10,93],"align":[11],"with":[12,269],"teachers'":[13,95,177,229],"goals,":[14],"preferences,":[15,96],"and":[16,109,124,162,181,208,275],"instructional":[17,277],"practices.":[18],"Teachers,":[19],"however,":[20,141],"have":[21,41,158,244,252],"scarce":[22],"time":[23,123],"to":[24,35,81,227,266],"make":[25],"such":[26,115],"customizations":[27],"themselves.":[28],"How":[29],"might":[30,55],"the":[31,59,83,146,150,168,185,215,234,241,248,262],"crowd":[32,60,242],"leveraged":[34],"help":[36],"time-strapped":[37],"teachers?":[38],"Crowdsourcing":[39],"pipelines":[40],"traditionally":[42],"focused":[43],"on":[44,145],"content":[45],"generation.":[46],"It":[47],"is":[48,220],"an":[49,72,88,116],"open":[50],"question":[51],"how":[52,265],"a":[53,64,76,221,258],"pipeline":[54,79,119],"designed":[57,80],"so":[58,92,240],"can":[61,243],"succeed":[62],"revision/customization":[65],"task.":[66],"In":[67,102],"this":[68],"paper,":[69],"we":[70,112],"explore":[71],"initial":[73],"version":[74],"of":[75,87,131,187,206,247,264],"teacher-guided":[77],"crowdsourcing":[78],"improve":[82,144],"adaptive":[84],"math":[85,107],"hints":[86,128,140,148,226],"tutoring":[90],"system":[91],"fit":[94],"while":[97],"requiring":[98],"minimal":[99,270],"expert":[100,271],"guidance.":[101],"two":[103,135],"experiments":[104],"involving":[105],"144":[106],"teachers":[108,198,251],"481":[110],"crowdworkers,":[111],"found":[113,196],"that":[114,167,197,218,250],"expert-guided":[117],"revision":[118],"could":[120],"save":[121],"experts'":[122],"produce":[125],"better":[126],"crowd-revised":[127],"(in":[129],"terms":[130],"teacher":[132],"satisfaction)":[133],"than":[134],"comparison":[136],"conditions.":[137],"The":[138,255],"revised":[139],"did":[142],"not":[143],"existing":[147],"AI":[151],"tutor,":[152],"which":[153],"were":[154],"carefully-written":[155],"but":[156],"still":[157],"room":[159],"for":[160,171,212,224,236,253],"improvement":[161],"customization.":[163],"Further":[164],"analysis":[165],"revealed":[166],"main":[169],"challenge":[170],"crowdworkers":[172],"lie":[174],"understanding":[176],"brief":[178],"written":[179],"comments":[180],"implementing":[182],"them":[183],"form":[186],"effective":[188],"edits,":[189],"without":[190],"introducing":[191],"new":[192],"problems.":[193],"We":[194],"also":[195,232],"preferred":[199],"their":[200],"own":[201],"revisions":[202],"over":[203],"other":[204],"sources":[205],"hints,":[207],"exhibited":[209],"varying":[210],"preferences":[211],"hints.":[213,254],"Overall,":[214],"results":[216],"confirm":[217],"there":[219],"clear":[222],"need":[223,235],"customizing":[225,276],"individual":[228],"preferences.":[230],"They":[231],"highlight":[233],"more":[237],"elaborate":[238],"scaffolds":[239],"specific":[245],"knowledge":[246],"requirements":[249],"study":[256],"represents":[257],"first":[259],"exploration":[260],"literature":[263],"support":[267],"crowds":[268],"guidance":[272],"revising":[274],"materials.":[278]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":3},{"year":2021,"cited_by_count":2}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}