{"id":"https://openalex.org/W3207603166","doi":"https://doi.org/10.1145/3479863","title":"The UX Factor: Using Comparative Peer Review to Evaluate Designs through User Preferences","display_name":"The UX Factor: Using Comparative Peer Review to Evaluate Designs through User Preferences","publication_year":2021,"publication_date":"2021-10-13","ids":{"openalex":"https://openalex.org/W3207603166","doi":"https://doi.org/10.1145/3479863","mag":"3207603166"},"language":"en","primary_location":{"id":"doi:10.1145/3479863","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3479863","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3479863","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3479863","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5018601561","display_name":"Sarah Bawabe","orcid":null},"institutions":[{"id":"https://openalex.org/I27804330","display_name":"Brown University","ror":"https://ror.org/05gq02987","country_code":"US","type":"education","lineage":["https://openalex.org/I27804330"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Sarah Bawabe","raw_affiliation_strings":["Brown University, Providence, RI, USA"],"affiliations":[{"raw_affiliation_string":"Brown University, Providence, RI, USA","institution_ids":["https://openalex.org/I27804330"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050583996","display_name":"Laura Wilson","orcid":"https://orcid.org/0000-0001-9756-0845"},"institutions":[{"id":"https://openalex.org/I27804330","display_name":"Brown University","ror":"https://ror.org/05gq02987","country_code":"US","type":"education","lineage":["https://openalex.org/I27804330"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Laura Wilson","raw_affiliation_strings":["Brown University, Providence, RI, USA"],"affiliations":[{"raw_affiliation_string":"Brown University, Providence, RI, USA","institution_ids":["https://openalex.org/I27804330"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059747729","display_name":"Tongyu Zhou","orcid":"https://orcid.org/0000-0003-4003-518X"},"institutions":[{"id":"https://openalex.org/I27804330","display_name":"Brown University","ror":"https://ror.org/05gq02987","country_code":"US","type":"education","lineage":["https://openalex.org/I27804330"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tongyu Zhou","raw_affiliation_strings":["Brown University, Providence, RI, USA"],"affiliations":[{"raw_affiliation_string":"Brown University, Providence, RI, USA","institution_ids":["https://openalex.org/I27804330"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021559956","display_name":"Ezra Marks","orcid":null},"institutions":[{"id":"https://openalex.org/I27804330","display_name":"Brown University","ror":"https://ror.org/05gq02987","country_code":"US","type":"education","lineage":["https://openalex.org/I27804330"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ezra Marks","raw_affiliation_strings":["Brown University, Providence, RI, USA"],"affiliations":[{"raw_affiliation_string":"Brown University, Providence, RI, USA","institution_ids":["https://openalex.org/I27804330"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5052381120","display_name":"Jeff Huang","orcid":"https://orcid.org/0000-0002-3453-5666"},"institutions":[{"id":"https://openalex.org/I27804330","display_name":"Brown University","ror":"https://ror.org/05gq02987","country_code":"US","type":"education","lineage":["https://openalex.org/I27804330"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jeff Huang","raw_affiliation_strings":["Brown University, Providence, RI, USA"],"affiliations":[{"raw_affiliation_string":"Brown University, Providence, RI, USA","institution_ids":["https://openalex.org/I27804330"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5018601561"],"corresponding_institution_ids":["https://openalex.org/I27804330"],"apc_list":null,"apc_paid":null,"fwci":0.9442,"has_fulltext":true,"cited_by_count":7,"citation_normalized_percentile":{"value":0.8039383,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":98},"biblio":{"volume":"5","issue":"CSCW2","first_page":"1","last_page":"23"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10636","display_name":"Innovative Teaching and Learning Methods","score":0.9771000146865845,"subfield":{"id":"https://openalex.org/subfields/3204","display_name":"Developmental and Educational Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11039","display_name":"Evaluation of Teaching Practices","score":0.96670001745224,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7053132653236389},{"id":"https://openalex.org/keywords/rubric","display_name":"Rubric","score":0.6602894067764282},{"id":"https://openalex.org/keywords/grading","display_name":"Grading (engineering)","score":0.5700938701629639},{"id":"https://openalex.org/keywords/peer-assessment","display_name":"Peer assessment","score":0.5121145248413086},{"id":"https://openalex.org/keywords/peer-feedback","display_name":"Peer feedback","score":0.47974956035614014},{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"Pairwise comparison","score":0.4575045108795166},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4355800449848175},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.4312129616737366},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.25382471084594727},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.21822407841682434},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.17752733826637268},{"id":"https://openalex.org/keywords/mathematics-education","display_name":"Mathematics education","score":0.16555452346801758},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.1048550009727478}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7053132653236389},{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.6602894067764282},{"id":"https://openalex.org/C2777286243","wikidata":"https://www.wikidata.org/wiki/Q5591926","display_name":"Grading (engineering)","level":2,"score":0.5700938701629639},{"id":"https://openalex.org/C53839665","wikidata":"https://www.wikidata.org/wiki/Q2067088","display_name":"Peer assessment","level":2,"score":0.5121145248413086},{"id":"https://openalex.org/C42058189","wikidata":"https://www.wikidata.org/wiki/Q7160400","display_name":"Peer feedback","level":2,"score":0.47974956035614014},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.4575045108795166},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4355800449848175},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.4312129616737366},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.25382471084594727},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.21822407841682434},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.17752733826637268},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.16555452346801758},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.1048550009727478},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C147176958","wikidata":"https://www.wikidata.org/wiki/Q77590","display_name":"Civil engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3479863","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3479863","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3479863","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3479863","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3479863","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3479863","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.800000011920929}],"awards":[{"id":"https://openalex.org/G1291400521","display_name":null,"funder_award_id":"1552663","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6159144427","display_name":null,"funder_award_id":"IIS-1552663","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320309070","display_name":"Brown University","ror":"https://ror.org/05gq02987"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3207603166.pdf","grobid_xml":"https://content.openalex.org/works/W3207603166.grobid-xml"},"referenced_works_count":37,"referenced_works":["https://openalex.org/W234809696","https://openalex.org/W1866122539","https://openalex.org/W1985858262","https://openalex.org/W1992714945","https://openalex.org/W1994213885","https://openalex.org/W1996469193","https://openalex.org/W2003324563","https://openalex.org/W2024583961","https://openalex.org/W2062680454","https://openalex.org/W2073271971","https://openalex.org/W2084422199","https://openalex.org/W2088700588","https://openalex.org/W2100588183","https://openalex.org/W2102214098","https://openalex.org/W2122588206","https://openalex.org/W2124645092","https://openalex.org/W2141413057","https://openalex.org/W2164545125","https://openalex.org/W2165181820","https://openalex.org/W2166164009","https://openalex.org/W2219614165","https://openalex.org/W2252693249","https://openalex.org/W2336125216","https://openalex.org/W2339932976","https://openalex.org/W2400788632","https://openalex.org/W2605734857","https://openalex.org/W2606187948","https://openalex.org/W2756771427","https://openalex.org/W2763110165","https://openalex.org/W2770197309","https://openalex.org/W2792183666","https://openalex.org/W2795596428","https://openalex.org/W2953342260","https://openalex.org/W2963131915","https://openalex.org/W3046252086","https://openalex.org/W4251548474","https://openalex.org/W4298204463"],"related_works":["https://openalex.org/W2525485641","https://openalex.org/W2488232887","https://openalex.org/W1648438236","https://openalex.org/W3138734495","https://openalex.org/W2610634284","https://openalex.org/W3132811443","https://openalex.org/W2992816059","https://openalex.org/W2912039502","https://openalex.org/W2742817363","https://openalex.org/W2596737561"],"abstract_inverted_index":{"Peer":[0],"review":[1,172],"has":[2],"been":[3],"used":[4,61],"in":[5,85,113],"both":[6,146],"online":[7],"and":[8,16,32,54,77,93,106,127,150,162],"offline":[9],"classrooms":[10],"to":[11,47,62,74,145,159,173],"inspire":[12],"creativity,":[13],"gather":[14],"feedback,":[15,36],"lessen":[17],"instructor":[18],"grading":[19,43],"loads,":[20],"especially":[21],"for":[22],"design-based":[23],"tasks":[24],"without":[25],"definitive":[26],"rubrics.":[27],"To":[28],"explore":[29],"the":[30,49,55,58,69,109,147,167,175],"nuances":[31],"quality":[33,176],"of":[34,51,57,71,90,138,169,177],"peer":[35,42,52,171],"we":[37],"developed":[38],"UX":[39,83],"Factor,":[40],"a":[41,86,104,135],"platform":[44],"that":[45,108,119,142],"aims":[46],"characterize":[48],"behavior":[50],"reviews":[53],"consistency":[56],"ranking":[59],"models":[60],"aggregate":[63],"these":[64],"reviews.":[65],"This":[66],"system":[67,110],"harnesses":[68],"power":[70],"pairwise":[72],"comparisons":[73],"minimize":[75],"bias":[76],"encourage":[78],"context-driven":[79],"analysis.":[80],"We":[81,117],"adopted":[82],"Factor":[84],"user":[87],"interface":[88],"course":[89],"133":[91],"students":[92],"teaching":[94],"assistants":[95],"(TAs)":[96],"across":[97],"3":[98],"different":[99],"individual":[100],"design":[101],"projects":[102],"over":[103],"semester":[105],"found":[107],"was":[111],"effective":[112],"eliciting":[114],"high-quality":[115],"feedback.":[116],"saw":[118],"raters":[120,161],"have":[121],"higher":[122],"agreement":[123],"than":[124],"random":[125],"preferences,":[126],"with":[128],"at":[129,180],"least":[130],"15":[131],"ratings":[132,139,149],"per":[133],"submission,":[134],"simple":[136],"average":[137],"produced":[140],"rankings":[141,156],"were":[143,157],"consistent":[144],"raw":[148],"other":[151],"more":[152],"complex":[153],"models.":[154],"These":[155],"robust":[158],"disagreeable":[160],"changing":[163],"class":[164],"sizes,":[165],"demonstrating":[166],"potential":[168],"comparative":[170],"match":[174],"expert":[178],"feedback":[179],"scale.":[181]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":2}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}