{"id":"https://openalex.org/W2606502823","doi":"https://doi.org/10.1145/3051457.3051463","title":"Robust Evaluation Matrix","display_name":"Robust Evaluation Matrix","publication_year":2017,"publication_date":"2017-04-12","ids":{"openalex":"https://openalex.org/W2606502823","doi":"https://doi.org/10.1145/3051457.3051463","mag":"2606502823"},"language":"en","primary_location":{"id":"doi:10.1145/3051457.3051463","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3051457.3051463","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Fourth (2017) ACM Conference on Learning @ Scale","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5064001395","display_name":"Shayan Doroudi","orcid":"https://orcid.org/0000-0002-0602-1406"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Shayan Doroudi","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047522207","display_name":"Vincent Aleven","orcid":"https://orcid.org/0000-0002-1581-6657"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Vincent Aleven","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5084989076","display_name":"Emma Brunskill","orcid":"https://orcid.org/0000-0002-3971-7127"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Emma Brunskill","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5064001395"],"corresponding_institution_ids":["https://openalex.org/I74973139"],"apc_list":null,"apc_paid":null,"fwci":1.1701,"has_fulltext":false,"cited_by_count":12,"citation_normalized_percentile":{"value":0.8366751,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"3","last_page":"12"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.9901000261306763,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.9901000261306763,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.9861000180244446,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9627000093460083,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5436354279518127},{"id":"https://openalex.org/keywords/matrix","display_name":"Matrix (chemical analysis)","score":0.42257049679756165},{"id":"https://openalex.org/keywords/materials-science","display_name":"Materials science","score":0.0875263512134552}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5436354279518127},{"id":"https://openalex.org/C106487976","wikidata":"https://www.wikidata.org/wiki/Q685816","display_name":"Matrix (chemical analysis)","level":2,"score":0.42257049679756165},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0875263512134552},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3051457.3051463","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3051457.3051463","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Fourth (2017) ACM Conference on Learning @ Scale","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","score":0.699999988079071,"id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G8256617804","display_name":null,"funder_award_id":"R305A130215,R305B150008","funder_id":"https://openalex.org/F4320332210","funder_display_name":"Institute of Education Sciences"}],"funders":[{"id":"https://openalex.org/F4320332210","display_name":"Institute of Education Sciences","ror":"https://ror.org/04et59085"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":28,"referenced_works":["https://openalex.org/W169621967","https://openalex.org/W650350307","https://openalex.org/W1184128069","https://openalex.org/W1483702380","https://openalex.org/W1514587017","https://openalex.org/W1525421687","https://openalex.org/W1596401170","https://openalex.org/W1809653203","https://openalex.org/W1859820673","https://openalex.org/W2001468118","https://openalex.org/W2015040676","https://openalex.org/W2105715011","https://openalex.org/W2108692343","https://openalex.org/W2114904200","https://openalex.org/W2138331287","https://openalex.org/W2160178500","https://openalex.org/W2168923397","https://openalex.org/W2188328586","https://openalex.org/W2251007294","https://openalex.org/W2273088453","https://openalex.org/W2294691030","https://openalex.org/W2403755232","https://openalex.org/W2466877100","https://openalex.org/W2467237061","https://openalex.org/W2573261414","https://openalex.org/W3098679278","https://openalex.org/W4247950230","https://openalex.org/W4253533180"],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W2382290278","https://openalex.org/W2478288626","https://openalex.org/W4391913857","https://openalex.org/W2350741829","https://openalex.org/W2530322880"],"abstract_inverted_index":{"The":[0],"gold":[1],"standard":[2],"for":[3,150],"identifying":[4],"more":[5],"effective":[6,236,247],"pedagogical":[7],"approaches":[8],"is":[9,72,84,103,124],"to":[10,43,55,87,119,126,136,180,191,226,241],"perform":[11],"an":[12,25,64],"experiment.":[13],"Unfortunately,":[14],"frequently":[15],"a":[16,61,89,96,189,204,211,229],"hypothesized":[17],"alternate":[18,112],"way":[19],"of":[20,33,40,59,92,111,140,157],"teaching":[21,66,113,145,159,183,194],"does":[22],"not":[23,234],"yield":[24],"improved":[26],"effect.":[27],"Given":[28],"the":[29,37,101,138,155,164,176,182,193],"expense":[30],"and":[31,36,185,209,240],"logistics":[32],"each":[34],"experiment,":[35],"enormous":[38],"space":[39],"potential":[41],"ways":[42,118],"improve":[44,69],"teaching,":[45],"it":[46,52,83,94,123,244],"would":[47,68],"be":[48,132,223,235,246],"highly":[49],"preferable":[50],"if":[51,82],"were":[53],"possible":[54,125],"estimate":[56,137],"in":[57,75,248],"advance":[58],"running":[60],"study":[62],"whether":[63],"alternative":[65],"strategy":[67,184,195],"learning.":[70,251],"This":[71],"true":[73],"even":[74,81],"learning":[76,213],"at":[77],"scale":[78],"situations,":[79],"since":[80],"logistically":[85],"easier":[86],"recruit":[88],"large":[90],"number":[91],"subjects,":[93],"remains":[95],"high":[97],"stakes":[98],"environment":[99],"because":[100],"experiment":[102],"impacting":[104],"many":[105],"real":[106],"students.":[107],"For":[108],"certain":[109],"classes":[110],"approaches,":[114],"such":[115],"as":[116,134,188],"new":[117,143,158,230],"sequence":[120],"existing":[121,148],"material,":[122],"build":[127],"student":[128,177,250],"models":[129],"that":[130,186,218],"can":[131,153],"used":[133,179,187,224],"simulators":[135],"performance":[139,156],"learners":[141],"under":[142],"proposed":[144],"methods.":[146,160],"However,":[147],"methods":[149],"doing":[151],"so":[152],"overestimate":[154],"We":[161,197],"instead":[162],"propose":[163],"Robust":[165],"Evaluation":[166],"Matrix":[167],"(REM)":[168],"method":[169],"which":[170],"explicitly":[171],"considers":[172],"model":[173,178],"mismatch":[174],"between":[175],"derive":[181],"simulator":[190],"evaluate":[192],"effectiveness.":[196],"then":[198],"present":[199],"two":[200],"case":[201],"studies":[202],"from":[203,210,215],"fractions":[205],"intelligent":[206],"tutoring":[207],"system":[208],"concept":[212],"task":[214],"prior":[216],"work":[217],"show":[219],"how":[220],"REM":[221],"could":[222],"both":[225],"detect":[227,242],"when":[228,243],"instructional":[231],"policy":[232],"may":[233,245],"on":[237],"actual":[238],"students":[239],"improving":[249]},"counts_by_year":[{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":5},{"year":2020,"cited_by_count":3},{"year":2019,"cited_by_count":1},{"year":2018,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}