{"id":"https://openalex.org/W2338609895","doi":"https://doi.org/10.1145/2876034.2893384","title":"Expert Evaluation of 300 Projects per Day","display_name":"Expert Evaluation of 300 Projects per Day","publication_year":2016,"publication_date":"2016-04-14","ids":{"openalex":"https://openalex.org/W2338609895","doi":"https://doi.org/10.1145/2876034.2893384","mag":"2338609895"},"language":"en","primary_location":{"id":"doi:10.1145/2876034.2893384","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2876034.2893384","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Third (2016) ACM Conference on Learning @ Scale","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103136135","display_name":"David Joyner","orcid":"https://orcid.org/0000-0003-0537-6229"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"David A. Joyner","raw_affiliation_strings":["Udacity, Inc., Mountain View, CA, USA"],"affiliations":[{"raw_affiliation_string":"Udacity, Inc., Mountain View, CA, USA","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5103136135"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.5526,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.7912473,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":94},"biblio":{"volume":null,"issue":null,"first_page":"121","last_page":"124"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9901999831199646,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9901999831199646,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.9387000203132629,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T14025","display_name":"Educational Technology and Assessment","score":0.9287999868392944,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6315423250198364},{"id":"https://openalex.org/keywords/grading","display_name":"Grading (engineering)","score":0.6113969087600708},{"id":"https://openalex.org/keywords/obstacle","display_name":"Obstacle","score":0.5930300951004028},{"id":"https://openalex.org/keywords/project-team","display_name":"Project team","score":0.4929921329021454},{"id":"https://openalex.org/keywords/engineering-management","display_name":"Engineering management","score":0.4704766869544983},{"id":"https://openalex.org/keywords/knowledge-management","display_name":"Knowledge management","score":0.23804661631584167},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.1871936023235321}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6315423250198364},{"id":"https://openalex.org/C2777286243","wikidata":"https://www.wikidata.org/wiki/Q5591926","display_name":"Grading (engineering)","level":2,"score":0.6113969087600708},{"id":"https://openalex.org/C2776650193","wikidata":"https://www.wikidata.org/wiki/Q264661","display_name":"Obstacle","level":2,"score":0.5930300951004028},{"id":"https://openalex.org/C2777205345","wikidata":"https://www.wikidata.org/wiki/Q1454317","display_name":"Project team","level":2,"score":0.4929921329021454},{"id":"https://openalex.org/C110354214","wikidata":"https://www.wikidata.org/wiki/Q6314146","display_name":"Engineering management","level":1,"score":0.4704766869544983},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.23804661631584167},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.1871936023235321},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C147176958","wikidata":"https://www.wikidata.org/wiki/Q77590","display_name":"Civil engineering","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2876034.2893384","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2876034.2893384","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Third (2016) ACM Conference on Learning @ Scale","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","score":0.44999998807907104,"display_name":"Industry, innovation and infrastructure"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":2,"referenced_works":["https://openalex.org/W2024807428","https://openalex.org/W2111940683"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2794103424","https://openalex.org/W1996530509","https://openalex.org/W3028317537","https://openalex.org/W2389515972","https://openalex.org/W4245435724","https://openalex.org/W2055301889","https://openalex.org/W4400979532","https://openalex.org/W2376554934"],"abstract_inverted_index":{"In":[0,127],"October":[1],"2014,":[2],"one-time":[3],"MOOC":[4],"developer":[5],"Udacity":[6,70],"completed":[7],"its":[8],"transition":[9,27],"from":[10],"primarily":[11],"producing":[12,18],"massive,":[13],"open":[14],"online":[15],"courses":[16],"to":[17],"job-focused,":[19],"project-based":[20,40],"microcredentials":[21,41],"called":[22],"\"Nanodegree\"":[23],"programs.":[24],"With":[25,49],"this":[26,84,128],"came":[28],"a":[29,59,72,91],"challenge:":[30],"whereas":[31],"MOOCs":[32],"focus":[33],"on":[34,90],"automated":[35],"assessment":[36],"and":[37,150],"peer-to-peer":[38],"grading,":[39],"would":[42],"only":[43],"be":[44],"feasible":[45],"with":[46,102],"expert":[47],"evaluation.":[48,66],"dreams":[50],"of":[51,54,56,113,134,142,147],"enrolling":[52],"tens":[53],"thousands":[55],"students":[57,117],"at":[58,125],"time,":[60],"the":[61,119,132,135,140,143,145,148,151],"major":[62],"obstacle":[63],"became":[64],"project":[65,79,88,136],"To":[67],"address":[68],"this,":[69],"developed":[71],"system":[73,85],"for":[74],"hiring":[75],"external":[76],"experts":[77],"as":[78],"reviewers.":[80],"A":[81],"year":[82],"later,":[83],"has":[86],"supported":[87],"evaluation":[89],"massive":[92],"scale:":[93],"61,000":[94],"projects":[95],"have":[96],"been":[97],"evaluated":[98,104],"in":[99],"12":[100],"months,":[101],"50%":[103],"within":[105,110],"2.5":[106],"hours":[107],"(and":[108],"88%":[109],"24":[111],"hours)":[112],"submission.":[114],"More":[115],"importantly,":[116],"rate":[118],"feedback":[120],"they":[121],"receive":[122],"very":[123],"highly":[124],"4.8/5.0.":[126],"paper,":[129],"we":[130],"discuss":[131],"structure":[133,146],"review":[137],"system,":[138],"including":[139],"nature":[141],"projects,":[144],"feedback,":[149],"data":[152],"described":[153],"above.":[154]},"counts_by_year":[{"year":2021,"cited_by_count":1},{"year":2017,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}