{"id":"https://openalex.org/W2988462106","doi":"https://doi.org/10.1145/3359242","title":"What's in a Review","display_name":"What's in a Review","publication_year":2019,"publication_date":"2019-11-07","ids":{"openalex":"https://openalex.org/W2988462106","doi":"https://doi.org/10.1145/3359242","mag":"2988462106"},"language":"en","primary_location":{"id":"doi:10.1145/3359242","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3359242","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3359242","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3359242","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5034011439","display_name":"Tiago Santos","orcid":"https://orcid.org/0000-0001-6030-774X"},"institutions":[{"id":"https://openalex.org/I4092182","display_name":"Graz University of Technology","ror":"https://ror.org/00d7xrm67","country_code":"AT","type":"education","lineage":["https://openalex.org/I4092182"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Tiago Santos","raw_affiliation_strings":["Graz University of Technology, Graz, Austria"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Graz University of Technology, Graz, Austria","institution_ids":["https://openalex.org/I4092182"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076202690","display_name":"Florian Lemmerich","orcid":"https://orcid.org/0000-0001-7620-1376"},"institutions":[{"id":"https://openalex.org/I887968799","display_name":"RWTH Aachen University","ror":"https://ror.org/04xfq0f34","country_code":"DE","type":"education","lineage":["https://openalex.org/I887968799"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Florian Lemmerich","raw_affiliation_strings":["RWTH Aachen University, Aachen, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"RWTH Aachen University, Aachen, Germany","institution_ids":["https://openalex.org/I887968799"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071422618","display_name":"Markus Strohmaier","orcid":"https://orcid.org/0000-0002-5485-5720"},"institutions":[{"id":"https://openalex.org/I887968799","display_name":"RWTH Aachen University","ror":"https://ror.org/04xfq0f34","country_code":"DE","type":"education","lineage":["https://openalex.org/I887968799"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Markus Strohmaier","raw_affiliation_strings":["RWTH Aachen University, Aachen, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"RWTH Aachen University, Aachen, Germany","institution_ids":["https://openalex.org/I887968799"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5074698656","display_name":"Denis Heli\u0107","orcid":"https://orcid.org/0000-0003-0725-7450"},"institutions":[{"id":"https://openalex.org/I4092182","display_name":"Graz University of Technology","ror":"https://ror.org/00d7xrm67","country_code":"AT","type":"education","lineage":["https://openalex.org/I4092182"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Denis Helic","raw_affiliation_strings":["Graz University of Technology, Graz, Austria"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Graz University of Technology, Graz, Austria","institution_ids":["https://openalex.org/I4092182"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.2767,"has_fulltext":true,"cited_by_count":26,"citation_normalized_percentile":{"value":0.85372532,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":"3","issue":"CSCW","first_page":"1","last_page":"22"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11574","display_name":"Artificial Intelligence in Games","score":0.9957000017166138,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11574","display_name":"Artificial Intelligence in Games","score":0.9957000017166138,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11197","display_name":"Digital Games and Media","score":0.994700014591217,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9925000071525574,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/amateur","display_name":"Amateur","score":0.8656362295150757},{"id":"https://openalex.org/keywords/video-game","display_name":"Video game","score":0.7851932644844055},{"id":"https://openalex.org/keywords/casual","display_name":"Casual","score":0.7022449970245361},{"id":"https://openalex.org/keywords/reputation","display_name":"Reputation","score":0.6433200836181641},{"id":"https://openalex.org/keywords/game-developer","display_name":"Game Developer","score":0.5335379838943481},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5031368136405945},{"id":"https://openalex.org/keywords/game-design","display_name":"Game design","score":0.4966183304786682},{"id":"https://openalex.org/keywords/video-game-design","display_name":"Video game design","score":0.44408655166625977},{"id":"https://openalex.org/keywords/realm","display_name":"Realm","score":0.4217597246170044},{"id":"https://openalex.org/keywords/game-mechanics","display_name":"Game mechanics","score":0.4199078679084778},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.3460147976875305},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.33995750546455383},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3299681544303894},{"id":"https://openalex.org/keywords/sociology","display_name":"Sociology","score":0.12890851497650146},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.10150444507598877},{"id":"https://openalex.org/keywords/social-science","display_name":"Social science","score":0.09469068050384521}],"concepts":[{"id":"https://openalex.org/C2778044066","wikidata":"https://www.wikidata.org/wiki/Q455595","display_name":"Amateur","level":2,"score":0.8656362295150757},{"id":"https://openalex.org/C3018412434","wikidata":"https://www.wikidata.org/wiki/Q7889","display_name":"Video game","level":2,"score":0.7851932644844055},{"id":"https://openalex.org/C2781426162","wikidata":"https://www.wikidata.org/wiki/Q2275793","display_name":"Casual","level":2,"score":0.7022449970245361},{"id":"https://openalex.org/C48798503","wikidata":"https://www.wikidata.org/wiki/Q877546","display_name":"Reputation","level":2,"score":0.6433200836181641},{"id":"https://openalex.org/C41826821","wikidata":"https://www.wikidata.org/wiki/Q844005","display_name":"Game Developer","level":3,"score":0.5335379838943481},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5031368136405945},{"id":"https://openalex.org/C503285160","wikidata":"https://www.wikidata.org/wiki/Q858057","display_name":"Game design","level":2,"score":0.4966183304786682},{"id":"https://openalex.org/C47187476","wikidata":"https://www.wikidata.org/wiki/Q12336660","display_name":"Video game design","level":3,"score":0.44408655166625977},{"id":"https://openalex.org/C2778757428","wikidata":"https://www.wikidata.org/wiki/Q1250464","display_name":"Realm","level":2,"score":0.4217597246170044},{"id":"https://openalex.org/C170828538","wikidata":"https://www.wikidata.org/wiki/Q1751513","display_name":"Game mechanics","level":2,"score":0.4199078679084778},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3460147976875305},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.33995750546455383},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3299681544303894},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.12890851497650146},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.10150444507598877},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.09469068050384521},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3359242","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3359242","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3359242","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},{"id":"pmh:oai:ub-madoc.bib.uni-mannheim.de:61651","is_oa":false,"landing_page_url":"https://madoc.bib.uni-mannheim.de/61651","pdf_url":null,"source":{"id":"https://openalex.org/S4377196315","display_name":"MADOC (University of Mannheim)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I177802217","host_organization_name":"University of Mannheim","host_organization_lineage":["https://openalex.org/I177802217"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Zeitschriftenartikel"}],"best_oa_location":{"id":"doi:10.1145/3359242","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3359242","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3359242","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.6499999761581421,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G6444727081","display_name":null,"funder_award_id":"DOC / 25005","funder_id":"https://openalex.org/F4320321004","funder_display_name":"\u00d6sterreichischen Akademie der Wissenschaften"}],"funders":[{"id":"https://openalex.org/F4320310430","display_name":"TU Graz, Internationale Beziehungen und Mobilit\u00e4tsprogramme","ror":"https://ror.org/00d7xrm67"},{"id":"https://openalex.org/F4320321004","display_name":"\u00d6sterreichischen Akademie der Wissenschaften","ror":"https://ror.org/03anc3s24"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2988462106.pdf","grobid_xml":"https://content.openalex.org/works/W2988462106.grobid-xml"},"referenced_works_count":77,"referenced_works":["https://openalex.org/W24166622","https://openalex.org/W818634956","https://openalex.org/W893486657","https://openalex.org/W998499328","https://openalex.org/W1482763710","https://openalex.org/W1491067015","https://openalex.org/W1494192578","https://openalex.org/W1506229096","https://openalex.org/W1647671624","https://openalex.org/W1783061888","https://openalex.org/W1880262756","https://openalex.org/W1968771487","https://openalex.org/W1977576157","https://openalex.org/W1993539390","https://openalex.org/W1993607151","https://openalex.org/W1993865115","https://openalex.org/W2001259128","https://openalex.org/W2009055572","https://openalex.org/W2030703717","https://openalex.org/W2047592681","https://openalex.org/W2050096199","https://openalex.org/W2055968785","https://openalex.org/W2057885236","https://openalex.org/W2061182607","https://openalex.org/W2061873838","https://openalex.org/W2064062304","https://openalex.org/W2070158055","https://openalex.org/W2080315483","https://openalex.org/W2085152852","https://openalex.org/W2086677051","https://openalex.org/W2087294982","https://openalex.org/W2099813784","https://openalex.org/W2101234009","https://openalex.org/W2102623335","https://openalex.org/W2103063352","https://openalex.org/W2104987199","https://openalex.org/W2134353060","https://openalex.org/W2142972908","https://openalex.org/W2152184085","https://openalex.org/W2152642979","https://openalex.org/W2155163959","https://openalex.org/W2156209975","https://openalex.org/W2159359879","https://openalex.org/W2163975393","https://openalex.org/W2167598575","https://openalex.org/W2169200297","https://openalex.org/W2170641288","https://openalex.org/W2217066517","https://openalex.org/W2287054277","https://openalex.org/W2338621248","https://openalex.org/W2408821365","https://openalex.org/W2416594539","https://openalex.org/W2424591430","https://openalex.org/W2575871963","https://openalex.org/W2583875861","https://openalex.org/W2736160752","https://openalex.org/W2788893025","https://openalex.org/W2802616634","https://openalex.org/W2808668898","https://openalex.org/W2949957935","https://openalex.org/W2963780471","https://openalex.org/W3101346006","https://openalex.org/W3105946648","https://openalex.org/W3106302634","https://openalex.org/W3122054972","https://openalex.org/W3122647499","https://openalex.org/W3122687407","https://openalex.org/W3122781176","https://openalex.org/W3123091093","https://openalex.org/W3123123052","https://openalex.org/W3124798136","https://openalex.org/W3124804010","https://openalex.org/W3126105808","https://openalex.org/W4230665241","https://openalex.org/W4254377434","https://openalex.org/W4289311243","https://openalex.org/W4297057888"],"related_works":["https://openalex.org/W4231050501","https://openalex.org/W4207081595","https://openalex.org/W4311459595","https://openalex.org/W1583852935","https://openalex.org/W2488706709","https://openalex.org/W4294947912","https://openalex.org/W2996376398","https://openalex.org/W2130088927","https://openalex.org/W4295008850","https://openalex.org/W3014566183"],"abstract_inverted_index":{"As":[0],"video":[1,14,55,90,111,194,259],"game":[2,15,56,91,118,159,195,211,260],"press":[3],"(\"experts\")":[4],"and":[5,50,60,62,70,96,120,174,209],"casual":[6],"gamers":[7],"(\"amateurs\")":[8],"have":[9],"different":[10],"motivations":[11],"when":[12,203,213],"writing":[13],"reviews,":[16],"discrepancies":[17,53],"in":[18,54,66,71,88,93,102,150,168,248,254],"their":[19],"reviews":[20,38,87,144,154,176],"may":[21],"arise.":[22],"To":[23],"study":[24,192],"such":[25],"potential":[26],"discrepancies,":[27],"we":[28,46,77,131,242],"conduct":[29],"a":[30,128,169],"large-scale":[31,190],"investigation":[32],"of":[33,52,82,110,158,193,231,233,238,246,252,258],"more":[34,107,146],"than":[35,141],"1":[36],"million":[37],"on":[39],"the":[40,48,79,94,151,188,229,234,239,244,250,255],"Metacritic":[41],"review":[42,72],"platform.":[43],"In":[44],"particular,":[45],"assess":[47],"existence":[49],"nature":[51],"appraisal":[57],"by":[58],"experts":[59,134,247],"amateurs,":[61,101,142],"how":[63],"they":[64],"manifest":[65],"ratings,":[67],"over":[68],"time,":[69],"language.":[73],"Leveraging":[74],"these":[75],"insights,":[76],"explore":[78],"predictive":[80,157],"power":[81],"early":[83],"expert":[84,173],"vs.":[85,236],"amateur":[86,153,175],"forecasting":[89],"reputation":[92,160],"short-":[95],"long-term.":[97],"We":[98],"find":[99],"that":[100,133],"contrast":[103],"to":[104,207,217,228],"experts,":[105],"give":[106],"polarized":[108],"ratings":[109],"games,":[112],"rate":[113],"games":[114,206,216],"surprisingly":[115],"long":[116],"after":[117],"release,":[119],"are":[121,145,155,177],"positively":[122],"biased":[123],"towards":[124],"older":[125],"games.":[126],"On":[127],"textual":[129],"level,":[130],"observe":[132],"write":[135],"rather":[136],"complex,":[137],"less":[138],"readable":[139],"texts":[140],"whose":[143],"emotionally":[147],"charged.":[148],"While":[149],"short-term":[152],"remarkably":[156],"among":[161],"other":[162],"amateurs":[163,202,253],"(achieving":[164],"91%":[165],"ROC":[166],"AUC":[167],"binary":[170],"classification),":[171],"both":[172],"equally":[178],"well":[179],"suited":[180],"for":[181,201,210],"long-term":[182],"predictions.":[183],"Overall,":[184],"our":[185,225],"work":[186,226],"is":[187],"first":[189],"comparative":[191],"reviewing":[196],"behavior,":[197],"with":[198],"practical":[199],"implications":[200],"deciding":[204],"which":[205,215],"play,":[208],"developers":[212],"planning":[214],"design,":[218],"develop,":[219],"or":[220],"continuously":[221],"support.":[222],"More":[223],"broadly,":[224],"contributes":[227],"discussion":[230],"wisdom":[232,237],"few":[235],"crowds,":[240],"as":[241],"uncover":[243],"limits":[245],"capturing":[249],"views":[251],"particular":[256],"context":[257],"reviews.":[261]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":5},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":5},{"year":2020,"cited_by_count":2}],"updated_date":"2026-07-25T09:21:30.201066","created_date":"2019-11-22T00:00:00"}