{"id":"https://openalex.org/W3207188120","doi":"https://doi.org/10.1145/3479576","title":"Challenges and Opportunities for Data-Centric Peer Evaluation Tools for Teamwork","display_name":"Challenges and Opportunities for Data-Centric Peer Evaluation Tools for Teamwork","publication_year":2021,"publication_date":"2021-10-13","ids":{"openalex":"https://openalex.org/W3207188120","doi":"https://doi.org/10.1145/3479576","mag":"3207188120"},"language":"en","primary_location":{"id":"doi:10.1145/3479576","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3479576","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3479576","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3479576","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5026774269","display_name":"Wenxuan Wendy Shi","orcid":"https://orcid.org/0000-0002-4217-803X"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Wenxuan Wendy Shi","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, Urbana, IL, USA"],"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, Urbana, IL, USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038067332","display_name":"Akshaya Jagannadharao","orcid":null},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Akshaya Jagannadharao","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, Urbana, IL, USA"],"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, Urbana, IL, USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100609347","display_name":"Jaewook Lee","orcid":"https://orcid.org/0000-0002-1481-9290"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jaewook Lee","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, Urbana, IL, USA"],"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, Urbana, IL, USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5034329641","display_name":"Brian P. Bailey","orcid":"https://orcid.org/0000-0003-1859-986X"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Brian P. Bailey","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, Urbana, IL, USA"],"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, Urbana, IL, USA","institution_ids":["https://openalex.org/I157725225"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5026774269"],"corresponding_institution_ids":["https://openalex.org/I157725225"],"apc_list":null,"apc_paid":null,"fwci":1.9158,"has_fulltext":true,"cited_by_count":10,"citation_normalized_percentile":{"value":0.85888006,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":98},"biblio":{"volume":"5","issue":"CSCW2","first_page":"1","last_page":"20"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10636","display_name":"Innovative Teaching and Learning Methods","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/3204","display_name":"Developmental and Educational Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10636","display_name":"Innovative Teaching and Learning Methods","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/3204","display_name":"Developmental and Educational Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10970","display_name":"Team Dynamics and Performance","score":0.9872000217437744,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.9732000231742859,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/teamwork","display_name":"Teamwork","score":0.8722922205924988},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.6080411672592163},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.5253260135650635},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5160238742828369},{"id":"https://openalex.org/keywords/work","display_name":"Work (physics)","score":0.4731244444847107},{"id":"https://openalex.org/keywords/peer-assessment","display_name":"Peer assessment","score":0.4353681206703186},{"id":"https://openalex.org/keywords/peer-review","display_name":"Peer review","score":0.42405837774276733},{"id":"https://openalex.org/keywords/knowledge-management","display_name":"Knowledge management","score":0.40848758816719055},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.35481375455856323},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.32107219099998474},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.19407272338867188},{"id":"https://openalex.org/keywords/mathematics-education","display_name":"Mathematics education","score":0.11524361371994019}],"concepts":[{"id":"https://openalex.org/C111226992","wikidata":"https://www.wikidata.org/wiki/Q626225","display_name":"Teamwork","level":2,"score":0.8722922205924988},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6080411672592163},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5253260135650635},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5160238742828369},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.4731244444847107},{"id":"https://openalex.org/C53839665","wikidata":"https://www.wikidata.org/wiki/Q2067088","display_name":"Peer assessment","level":2,"score":0.4353681206703186},{"id":"https://openalex.org/C138368954","wikidata":"https://www.wikidata.org/wiki/Q215028","display_name":"Peer review","level":2,"score":0.42405837774276733},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.40848758816719055},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.35481375455856323},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.32107219099998474},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.19407272338867188},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.11524361371994019},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3479576","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3479576","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3479576","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3479576","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3479576","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3479576","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G5350482788","display_name":null,"funder_award_id":"2016908","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8648400797","display_name":null,"funder_award_id":"IIS-2016908","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3207188120.pdf","grobid_xml":"https://content.openalex.org/works/W3207188120.grobid-xml"},"referenced_works_count":42,"referenced_works":["https://openalex.org/W73973345","https://openalex.org/W604972776","https://openalex.org/W1968587910","https://openalex.org/W1973160432","https://openalex.org/W1977555871","https://openalex.org/W1983799571","https://openalex.org/W2010939781","https://openalex.org/W2037812148","https://openalex.org/W2038924341","https://openalex.org/W2041573082","https://openalex.org/W2043062223","https://openalex.org/W2051075990","https://openalex.org/W2088559924","https://openalex.org/W2094151552","https://openalex.org/W2095006107","https://openalex.org/W2097788690","https://openalex.org/W2105918120","https://openalex.org/W2112865564","https://openalex.org/W2124100711","https://openalex.org/W2130306321","https://openalex.org/W2139622443","https://openalex.org/W2140236135","https://openalex.org/W2161846679","https://openalex.org/W2170038885","https://openalex.org/W2171636554","https://openalex.org/W2400788632","https://openalex.org/W2411190811","https://openalex.org/W2524605839","https://openalex.org/W2562606560","https://openalex.org/W2593151843","https://openalex.org/W2613995098","https://openalex.org/W2617573081","https://openalex.org/W2765596726","https://openalex.org/W2775528210","https://openalex.org/W2795596428","https://openalex.org/W2796286572","https://openalex.org/W2796420997","https://openalex.org/W2941298121","https://openalex.org/W2979902769","https://openalex.org/W3021174912","https://openalex.org/W3039090620","https://openalex.org/W3136116058"],"related_works":["https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W4293766277","https://openalex.org/W2344285731","https://openalex.org/W3048141256","https://openalex.org/W3112412740","https://openalex.org/W2168973021","https://openalex.org/W4287691201","https://openalex.org/W1517428334","https://openalex.org/W2569496898"],"abstract_inverted_index":{"Peer":[0],"evaluations":[1,137],"are":[2,8,33],"critical":[3],"for":[4,55,184],"assessing":[5],"teams,":[6],"but":[7,97],"susceptible":[9],"to":[10,29,89,106,116,152],"bias":[11],"and":[12,47,52,65,69,103,121,125,143,158,163,180],"other":[13],"factors":[14],"that":[15,25,39,79],"undermine":[16],"their":[17,31,95,136],"reliability.":[18],"At":[19],"the":[20,83,90,108,153,164,168,171],"same":[21],"time,":[22],"collaborative":[23,133],"tools":[24,134,157],"teams":[26,162],"commonly":[27],"use":[28],"perform":[30],"work":[32],"increasingly":[34],"capable":[35],"of":[36,82,114,138,156,170],"logging":[37],"activity":[38,56,130,149],"can":[40],"signal":[41],"useful":[42],"information":[43],"about":[44,147],"individual":[45],"contributions":[46,88],"teamwork.":[48],"To":[49],"investigate":[50],"current":[51],"potential":[53],"uses":[54],"traces":[57,131,150],"in":[58],"peer":[59,123,187],"evaluation":[60,188],"tools,":[61],"we":[62,177],"interviewed":[63,70],"(N=11)":[64],"surveyed":[66,85],"(N=242)":[67],"students":[68,84,142],"(N=10)":[71],"instructors":[72,144],"at":[73],"a":[74,181,185],"single":[75],"university.":[76],"We":[77],"found":[78],"nearly":[80],"all":[81],"considered":[86],"specific":[87],"team":[91],"outcomes":[92],"when":[93],"evaluating":[94],"teammates,":[96],"also":[98],"reported":[99],"relying":[100],"on":[101,174],"memory":[102],"subjective":[104],"experiences":[105],"make":[107],"assessment.":[109],"Instructors":[110],"desired":[111],"objective":[112],"sources":[113],"data":[115],"address":[117],"challenges":[118],"with":[119],"administering":[120],"interpreting":[122],"evaluations,":[124],"have":[126],"already":[127],"begun":[128],"incorporating":[129],"from":[132],"into":[135,167],"teams.":[139],"However,":[140],"both":[141],"expressed":[145],"concern":[146],"using":[148],"due":[151],"diverse":[154],"ecosystem":[155],"platforms":[159],"used":[160],"by":[161],"limited":[165],"view":[166],"context":[169],"contributions.":[172],"Based":[173],"our":[175],"findings,":[176],"contribute":[178],"recommendations":[179],"speculative":[182],"design":[183],"data-centric":[186],"tool.":[189]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":2}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}