{"id":"https://openalex.org/W2336125216","doi":"https://doi.org/10.1145/2876034.2893424","title":"Peer Reviewing Short Answers using Comparative Judgement","display_name":"Peer Reviewing Short Answers using Comparative Judgement","publication_year":2016,"publication_date":"2016-04-14","ids":{"openalex":"https://openalex.org/W2336125216","doi":"https://doi.org/10.1145/2876034.2893424","mag":"2336125216"},"language":"en","primary_location":{"id":"doi:10.1145/2876034.2893424","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2876034.2893424","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Third (2016) ACM Conference on Learning @ Scale","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5010208377","display_name":"Pushkar Kolhe","orcid":null},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Pushkar Kolhe","raw_affiliation_strings":["Georgia Institute of Technology, Atlanta, GA, USA"],"affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, Atlanta, GA, USA","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009722403","display_name":"Michael L. Littman","orcid":"https://orcid.org/0000-0002-5596-1840"},"institutions":[{"id":"https://openalex.org/I196272386","display_name":"Providence College","ror":"https://ror.org/00rxpqe74","country_code":"US","type":"education","lineage":["https://openalex.org/I196272386"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michael L. Littman","raw_affiliation_strings":["Computer Science, Providence, RI, USA"],"affiliations":[{"raw_affiliation_string":"Computer Science, Providence, RI, USA","institution_ids":["https://openalex.org/I196272386"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5091212723","display_name":"Charles L. Isbell","orcid":null},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Charles L. Isbell","raw_affiliation_strings":["Georgia Institute of Technology, Atlanta, GA, USA"],"affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, Atlanta, GA, USA","institution_ids":["https://openalex.org/I130701444"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5010208377"],"corresponding_institution_ids":["https://openalex.org/I130701444"],"apc_list":null,"apc_paid":null,"fwci":2.654,"has_fulltext":false,"cited_by_count":11,"citation_normalized_percentile":{"value":0.91404979,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"241","last_page":"244"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9937000274658203,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9937000274658203,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14025","display_name":"Educational Technology and Assessment","score":0.9739000201225281,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9728000164031982,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/judgement","display_name":"Judgement","score":0.8994737863540649},{"id":"https://openalex.org/keywords/grading","display_name":"Grading (engineering)","score":0.8257921934127808},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6883112788200378},{"id":"https://openalex.org/keywords/scheme","display_name":"Scheme (mathematics)","score":0.6053810119628906},{"id":"https://openalex.org/keywords/probabilistic-logic","display_name":"Probabilistic logic","score":0.6022096276283264},{"id":"https://openalex.org/keywords/class","display_name":"Class (philosophy)","score":0.43000757694244385},{"id":"https://openalex.org/keywords/mathematics-education","display_name":"Mathematics education","score":0.3994537889957428},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.38750696182250977},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.16936242580413818},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.14671900868415833},{"id":"https://openalex.org/keywords/epistemology","display_name":"Epistemology","score":0.11293289065361023},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.05970484018325806}],"concepts":[{"id":"https://openalex.org/C2776548248","wikidata":"https://www.wikidata.org/wiki/Q12621536","display_name":"Judgement","level":2,"score":0.8994737863540649},{"id":"https://openalex.org/C2777286243","wikidata":"https://www.wikidata.org/wiki/Q5591926","display_name":"Grading (engineering)","level":2,"score":0.8257921934127808},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6883112788200378},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.6053810119628906},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.6022096276283264},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.43000757694244385},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.3994537889957428},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38750696182250977},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.16936242580413818},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.14671900868415833},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.11293289065361023},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.05970484018325806},{"id":"https://openalex.org/C147176958","wikidata":"https://www.wikidata.org/wiki/Q77590","display_name":"Civil engineering","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2876034.2893424","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2876034.2893424","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Third (2016) ACM Conference on Learning @ Scale","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.8799999952316284,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":6,"referenced_works":["https://openalex.org/W2022710553","https://openalex.org/W2054475738","https://openalex.org/W2083570596","https://openalex.org/W2122588206","https://openalex.org/W2166164009","https://openalex.org/W4229635495"],"related_works":["https://openalex.org/W2886802431","https://openalex.org/W4388216822","https://openalex.org/W1761762290","https://openalex.org/W4391191813","https://openalex.org/W4387575966","https://openalex.org/W2384262901","https://openalex.org/W2975214487","https://openalex.org/W651304006","https://openalex.org/W2345720417","https://openalex.org/W2385564112"],"abstract_inverted_index":{"We":[0,41],"propose":[1],"a":[2,27,47],"comparative":[3],"judgement":[4],"scheme":[5,16,53],"for":[6,51,61],"grading":[7,64],"short":[8,23],"answer":[9,22,24],"questions":[10],"in":[11],"an":[12],"online":[13],"class.":[14],"The":[15],"works":[17],"by":[18,39],"asking":[19],"students":[20,59],"to":[21],"questions.":[25,65],"Then":[26],"multiple":[28],"choice":[29],"question":[30],"is":[31],"created":[32],"whose":[33],"choices":[34],"are":[35],"the":[36],"answers":[37],"given":[38],"students.":[40],"show":[42],"that":[43],"we":[44],"can":[45],"formulate":[46],"probabilistic":[48],"graphical":[49],"model":[50],"this":[52],"which":[54],"lets":[55],"us":[56],"infer":[57],"each":[58],"proficiency":[60],"answering":[62],"and":[63]},"counts_by_year":[{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":1},{"year":2021,"cited_by_count":3},{"year":2019,"cited_by_count":2},{"year":2018,"cited_by_count":1},{"year":2017,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}