{"id":"https://openalex.org/W4366003132","doi":"https://doi.org/10.1145/3579627","title":"The Value of Activity Traces in Peer Evaluations: An Experimental Study","display_name":"The Value of Activity Traces in Peer Evaluations: An Experimental Study","publication_year":2023,"publication_date":"2023-04-14","ids":{"openalex":"https://openalex.org/W4366003132","doi":"https://doi.org/10.1145/3579627"},"language":"en","primary_location":{"id":"doi:10.1145/3579627","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3579627","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3579627","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3579627","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5026774269","display_name":"Wenxuan Wendy Shi","orcid":"https://orcid.org/0000-0002-4217-803X"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wenxuan Wendy Shi","raw_affiliation_strings":["University of Illinois, Urbana-Champaign, Champaign, IL, USA"],"raw_orcid":"https://orcid.org/0000-0002-4217-803X","affiliations":[{"raw_affiliation_string":"University of Illinois, Urbana-Champaign, Champaign, IL, USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069047639","display_name":"Sneha R. Krishna Kumaran","orcid":"https://orcid.org/0000-0001-7211-5665"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sneha R. Krishna Kumaran","raw_affiliation_strings":["University of Illinois, Urbana Champaign, Urbana, IL, USA"],"raw_orcid":"https://orcid.org/0000-0001-7211-5665","affiliations":[{"raw_affiliation_string":"University of Illinois, Urbana Champaign, Urbana, IL, USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018532037","display_name":"Hari Sundaram","orcid":"https://orcid.org/0000-0003-3315-6055"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hari Sundaram","raw_affiliation_strings":["University of Illinois, Urbana-Champaign, Urbana, IL, USA"],"raw_orcid":"https://orcid.org/0000-0003-3315-6055","affiliations":[{"raw_affiliation_string":"University of Illinois, Urbana-Champaign, Urbana, IL, USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5034329641","display_name":"Brian P. Bailey","orcid":"https://orcid.org/0000-0003-1859-986X"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Brian P. Bailey","raw_affiliation_strings":["University of Illinois, Urbana-Champaign, Urbana, IL, USA"],"raw_orcid":"https://orcid.org/0000-0003-1859-986X","affiliations":[{"raw_affiliation_string":"University of Illinois, Urbana-Champaign, Urbana, IL, USA","institution_ids":["https://openalex.org/I157725225"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I157725225"],"apc_list":null,"apc_paid":null,"fwci":2.2672,"has_fulltext":true,"cited_by_count":10,"citation_normalized_percentile":{"value":0.90126398,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"7","issue":"CSCW1","first_page":"1","last_page":"39"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12028","display_name":"Knowledge Management and Sharing","score":0.9962000250816345,"subfield":{"id":"https://openalex.org/subfields/3315","display_name":"Communication"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10636","display_name":"Innovative Teaching and Learning Methods","score":0.9955999851226807,"subfield":{"id":"https://openalex.org/subfields/3204","display_name":"Developmental and Educational Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/teamwork","display_name":"Teamwork","score":0.6734945774078369},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.6262621283531189},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.506435751914978},{"id":"https://openalex.org/keywords/value","display_name":"Value (mathematics)","score":0.5000936985015869},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.4943702220916748},{"id":"https://openalex.org/keywords/peer-evaluation","display_name":"Peer evaluation","score":0.48962265253067017},{"id":"https://openalex.org/keywords/peer-assessment","display_name":"Peer assessment","score":0.4826853275299072},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.40536826848983765},{"id":"https://openalex.org/keywords/applied-psychology","display_name":"Applied psychology","score":0.3658880591392517},{"id":"https://openalex.org/keywords/knowledge-management","display_name":"Knowledge management","score":0.3389016389846802},{"id":"https://openalex.org/keywords/higher-education","display_name":"Higher education","score":0.18315574526786804},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.10835358500480652}],"concepts":[{"id":"https://openalex.org/C111226992","wikidata":"https://www.wikidata.org/wiki/Q626225","display_name":"Teamwork","level":2,"score":0.6734945774078369},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.6262621283531189},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.506435751914978},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.5000936985015869},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4943702220916748},{"id":"https://openalex.org/C2988746165","wikidata":"https://www.wikidata.org/wiki/Q215028","display_name":"Peer evaluation","level":3,"score":0.48962265253067017},{"id":"https://openalex.org/C53839665","wikidata":"https://www.wikidata.org/wiki/Q2067088","display_name":"Peer assessment","level":2,"score":0.4826853275299072},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.40536826848983765},{"id":"https://openalex.org/C75630572","wikidata":"https://www.wikidata.org/wiki/Q538904","display_name":"Applied psychology","level":1,"score":0.3658880591392517},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.3389016389846802},{"id":"https://openalex.org/C120912362","wikidata":"https://www.wikidata.org/wiki/Q136822","display_name":"Higher education","level":2,"score":0.18315574526786804},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.10835358500480652},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.0},{"id":"https://openalex.org/C19417346","wikidata":"https://www.wikidata.org/wiki/Q7922","display_name":"Pedagogy","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3579627","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3579627","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3579627","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3579627","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3579627","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3579627","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G5350482788","display_name":"Sociotechnical Interventions for Nurturing Successful Team Learning Experiences","funder_award_id":"2016908","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8648400797","display_name":null,"funder_award_id":"IIS-2016908","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4366003132.pdf","grobid_xml":"https://content.openalex.org/works/W4366003132.grobid-xml"},"referenced_works_count":37,"referenced_works":["https://openalex.org/W73973345","https://openalex.org/W129285704","https://openalex.org/W604972776","https://openalex.org/W1968587910","https://openalex.org/W1974717757","https://openalex.org/W2010939781","https://openalex.org/W2032843080","https://openalex.org/W2043062223","https://openalex.org/W2051075990","https://openalex.org/W2070057526","https://openalex.org/W2086038483","https://openalex.org/W2088559924","https://openalex.org/W2094151552","https://openalex.org/W2095006107","https://openalex.org/W2106568252","https://openalex.org/W2112865564","https://openalex.org/W2124100711","https://openalex.org/W2137872373","https://openalex.org/W2139622443","https://openalex.org/W2142194139","https://openalex.org/W2161846679","https://openalex.org/W2171636554","https://openalex.org/W2177612807","https://openalex.org/W2342281051","https://openalex.org/W2404411786","https://openalex.org/W2544952174","https://openalex.org/W2562606560","https://openalex.org/W2580061943","https://openalex.org/W2589016589","https://openalex.org/W2617573081","https://openalex.org/W2796420997","https://openalex.org/W2897613819","https://openalex.org/W2897964068","https://openalex.org/W2898944906","https://openalex.org/W2979902769","https://openalex.org/W3207188120","https://openalex.org/W4232976691"],"related_works":["https://openalex.org/W1979384980","https://openalex.org/W2072584216","https://openalex.org/W2731835750","https://openalex.org/W3195190933","https://openalex.org/W3048141256","https://openalex.org/W4287691201","https://openalex.org/W2328132595","https://openalex.org/W17260207","https://openalex.org/W4255975601","https://openalex.org/W2972224096"],"abstract_inverted_index":{"Peer":[0],"evaluations":[1,92,120,138,160],"are":[2,16],"a":[3,74,123],"well-established":[4],"tool":[5],"for":[6,40,55,153,188],"evaluating":[7],"individual":[8],"and":[9,20,80,115,132,157,181,185],"team":[10,94],"performance":[11],"in":[12,96,175],"collaborative":[13,37,63,99],"contexts,":[14],"but":[15],"susceptible":[17],"to":[18,30,68,112,118,122],"social":[19],"cognitive":[21],"biases.":[22,43],"Current":[23],"peer":[24,56,91,159,194],"evaluation":[25,195],"tools":[26],"have":[27],"also":[28,128,179],"yet":[29],"address":[31],"the":[32,49,84,105,130,137,145],"unique":[33],"opportunities":[34],"that":[35,104,139],"online":[36,98],"technologies":[38],"provide":[39],"addressing":[41],"these":[42],"In":[44],"this":[45],"work,":[46],"we":[47,66,178],"explore":[48],"potential":[50],"of":[51,86,93,107,136,147,161,172],"one":[52],"such":[53],"opportunity":[54],"evaluations:":[57],"data":[58],"traces":[59,89,109,149,191],"automatically":[60],"generated":[61],"by":[62],"tools,":[64],"which":[65],"refer":[67],"as":[69,150,167,169],"\"activity":[70],"traces\".":[71],"We":[72],"conduct":[73],"between-subjects":[75],"experiment":[76],"with":[77],"101":[78],"students":[79],"MTurk":[81],"workers,":[82],"investigating":[83],"effects":[85],"reviewing":[87],"activity":[88,108,148,190],"on":[90,164],"members":[95],"an":[97,151],"task.":[100],"Our":[101,142],"findings":[102,143,166],"show":[103],"usage":[106],"led":[110],"participants":[111],"make":[113],"more":[114,155],"greater":[116],"revisions":[117,127],"their":[119],"compared":[121],"control":[124],"condition.":[125],"These":[126],"increased":[129],"consistency":[131],"participants'":[133],"perceived":[134],"accuracy":[135],"they":[140],"received.":[141],"demonstrate":[144],"value":[146],"approach":[152],"performing":[154],"reliable":[156],"objective":[158],"teamwork.":[162],"Based":[163],"our":[165,176],"well":[168],"qualitative":[170],"analysis":[171],"free-form":[173],"responses":[174],"study,":[177],"identify":[180],"discuss":[182],"key":[183],"considerations":[184],"design":[186],"recommendations":[187],"incorporating":[189],"into":[192],"real-world":[193],"systems.":[196]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":2}],"updated_date":"2026-07-31T08:31:51.225901","created_date":"2025-10-10T00:00:00"}