{"id":"https://openalex.org/W2068361436","doi":"https://doi.org/10.1145/1979742.1979788","title":"Weak inter-rater reliability in heuristic evaluation of video games","display_name":"Weak inter-rater reliability in heuristic evaluation of video games","publication_year":2011,"publication_date":"2011-05-07","ids":{"openalex":"https://openalex.org/W2068361436","doi":"https://doi.org/10.1145/1979742.1979788","mag":"2068361436"},"language":"en","primary_location":{"id":"doi:10.1145/1979742.1979788","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1979742.1979788","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"CHI '11 Extended Abstracts on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5090243381","display_name":"Gareth White","orcid":"https://orcid.org/0000-0002-1406-4983"},"institutions":[{"id":"https://openalex.org/I162608824","display_name":"University of Sussex","ror":"https://ror.org/00ayhx656","country_code":"GB","type":"education","lineage":["https://openalex.org/I162608824"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Gareth R. White","raw_affiliation_strings":["The University of Sussex, Brighton, United Kingdom","The University of Sussex, Brighton, United Kingdom#TAB#"],"affiliations":[{"raw_affiliation_string":"The University of Sussex, Brighton, United Kingdom","institution_ids":["https://openalex.org/I162608824"]},{"raw_affiliation_string":"The University of Sussex, Brighton, United Kingdom#TAB#","institution_ids":["https://openalex.org/I162608824"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072743620","display_name":"Pejman Mirza-Babaei","orcid":"https://orcid.org/0000-0002-8248-5392"},"institutions":[{"id":"https://openalex.org/I162608824","display_name":"University of Sussex","ror":"https://ror.org/00ayhx656","country_code":"GB","type":"education","lineage":["https://openalex.org/I162608824"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Pejman Mirza-babaei","raw_affiliation_strings":["The University of Sussex, Brighton, United Kingdom","The University of Sussex, Brighton, United Kingdom#TAB#"],"affiliations":[{"raw_affiliation_string":"The University of Sussex, Brighton, United Kingdom","institution_ids":["https://openalex.org/I162608824"]},{"raw_affiliation_string":"The University of Sussex, Brighton, United Kingdom#TAB#","institution_ids":["https://openalex.org/I162608824"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5110133154","display_name":"Graham McAllister","orcid":null},"institutions":[{"id":"https://openalex.org/I162608824","display_name":"University of Sussex","ror":"https://ror.org/00ayhx656","country_code":"GB","type":"education","lineage":["https://openalex.org/I162608824"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Graham McAllister","raw_affiliation_strings":["The University of Sussex, Brighton, United Kingdom","The University of Sussex, Brighton, United Kingdom#TAB#"],"affiliations":[{"raw_affiliation_string":"The University of Sussex, Brighton, United Kingdom","institution_ids":["https://openalex.org/I162608824"]},{"raw_affiliation_string":"The University of Sussex, Brighton, United Kingdom#TAB#","institution_ids":["https://openalex.org/I162608824"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5087672338","display_name":"Judith Good","orcid":"https://orcid.org/0000-0002-8317-9346"},"institutions":[{"id":"https://openalex.org/I162608824","display_name":"University of Sussex","ror":"https://ror.org/00ayhx656","country_code":"GB","type":"education","lineage":["https://openalex.org/I162608824"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Judith Good","raw_affiliation_strings":["The University of Sussex, Brighton, United Kingdom","The University of Sussex, Brighton, United Kingdom#TAB#"],"affiliations":[{"raw_affiliation_string":"The University of Sussex, Brighton, United Kingdom","institution_ids":["https://openalex.org/I162608824"]},{"raw_affiliation_string":"The University of Sussex, Brighton, United Kingdom#TAB#","institution_ids":["https://openalex.org/I162608824"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5090243381"],"corresponding_institution_ids":["https://openalex.org/I162608824"],"apc_list":null,"apc_paid":null,"fwci":0.8739,"has_fulltext":false,"cited_by_count":18,"citation_normalized_percentile":{"value":0.75520331,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"1441","last_page":"1446"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10470","display_name":"Usability and User Interface Design","score":0.9947999715805054,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10470","display_name":"Usability and User Interface Design","score":0.9947999715805054,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9925000071525574,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.9847999811172485,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/heuristics","display_name":"Heuristics","score":0.9296853542327881},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.76897132396698},{"id":"https://openalex.org/keywords/heuristic","display_name":"Heuristic","score":0.7345162034034729},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6866083145141602},{"id":"https://openalex.org/keywords/heuristic-evaluation","display_name":"Heuristic evaluation","score":0.6577290892601013},{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.5678104758262634},{"id":"https://openalex.org/keywords/usability","display_name":"Usability","score":0.555016040802002},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.48465996980667114},{"id":"https://openalex.org/keywords/interpretation","display_name":"Interpretation (philosophy)","score":0.46560174226760864},{"id":"https://openalex.org/keywords/representativeness-heuristic","display_name":"Representativeness heuristic","score":0.44399991631507874},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.442515105009079},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.43919435143470764},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.4196047782897949},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.37187278270721436},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.27825701236724854},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.20045951008796692},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.16128602623939514}],"concepts":[{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.9296853542327881},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.76897132396698},{"id":"https://openalex.org/C173801870","wikidata":"https://www.wikidata.org/wiki/Q201413","display_name":"Heuristic","level":2,"score":0.7345162034034729},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6866083145141602},{"id":"https://openalex.org/C3255780","wikidata":"https://www.wikidata.org/wiki/Q1616517","display_name":"Heuristic evaluation","level":3,"score":0.6577290892601013},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.5678104758262634},{"id":"https://openalex.org/C170130773","wikidata":"https://www.wikidata.org/wiki/Q216378","display_name":"Usability","level":2,"score":0.555016040802002},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.48465996980667114},{"id":"https://openalex.org/C527412718","wikidata":"https://www.wikidata.org/wiki/Q855395","display_name":"Interpretation (philosophy)","level":2,"score":0.46560174226760864},{"id":"https://openalex.org/C37381756","wikidata":"https://www.wikidata.org/wiki/Q20203288","display_name":"Representativeness heuristic","level":2,"score":0.44399991631507874},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.442515105009079},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.43919435143470764},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.4196047782897949},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.37187278270721436},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.27825701236724854},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.20045951008796692},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.16128602623939514},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/1979742.1979788","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1979742.1979788","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"CHI '11 Extended Abstracts on Human Factors in Computing Systems","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":21,"referenced_works":["https://openalex.org/W137305461","https://openalex.org/W1495649899","https://openalex.org/W1506534339","https://openalex.org/W1590392599","https://openalex.org/W1977953443","https://openalex.org/W1980880315","https://openalex.org/W1982451263","https://openalex.org/W2004643557","https://openalex.org/W2005229107","https://openalex.org/W2035081519","https://openalex.org/W2047322855","https://openalex.org/W2061504941","https://openalex.org/W2075648114","https://openalex.org/W2078504914","https://openalex.org/W2088028234","https://openalex.org/W2099473180","https://openalex.org/W2101451656","https://openalex.org/W2148921999","https://openalex.org/W2160510495","https://openalex.org/W2162598207","https://openalex.org/W4233642407"],"related_works":["https://openalex.org/W2740053659","https://openalex.org/W2032280552","https://openalex.org/W158727920","https://openalex.org/W4292157289","https://openalex.org/W2912805850","https://openalex.org/W2116621310","https://openalex.org/W4288067675","https://openalex.org/W2096820616","https://openalex.org/W2912104648","https://openalex.org/W82190428"],"abstract_inverted_index":{"Heuristic":[0],"evaluation":[1,8],"promises":[2],"to":[3,48,105],"be":[4],"a":[5,19,53,57],"low-cost":[6],"usability":[7],"method,":[9],"but":[10],"is":[11,28,60,85,103],"fraught":[12],"with":[13,62,87],"problems":[14],"of":[15,21,34,56,90,94,96,109,132],"subjective":[16],"interpretation,":[17],"and":[18,23,119,124],"proliferation":[20],"competing":[22],"contradictory":[24],"heuristic":[25],"lists.":[26],"This":[27,100],"particularly":[29],"true":[30],"in":[31,65,113,129],"the":[32,50,97,106,122,126,133],"field":[33],"games":[35],"research":[36],"where":[37],"no":[38],"rigorous":[39],"comparative":[40],"validation":[41,93],"has":[42],"yet":[43],"been":[44],"published.":[45],"In":[46],"order":[47],"validate":[49],"available":[51,98],"heuristics,":[52],"user":[54],"test":[55],"commercial":[58],"game":[59],"conducted":[61],"6":[63],"participants":[64],"which":[66,72],"88":[67],"issues":[68],"are":[69,75],"identified,":[70],"against":[71],"146":[73],"heuristics":[74],"rated":[76],"for":[77,121],"relevance":[78],"by":[79],"3":[80],"evaluators.":[81],"Weak":[82],"inter-rater":[83],"reliability":[84,102],"calculated":[86],"Krippendorff's":[88],"Alpha":[89],"0.343,":[91],"refuting":[92],"any":[95],"heuristics.":[99,134],"weak":[101],"due":[104],"high":[107],"complexity":[108],"video":[110],"games,":[111],"resulting":[112],"evaluators":[114],"interpreting":[115],"different":[116],"reasonable":[117],"causes":[118],"solutions":[120],"issues,":[123],"hence":[125],"wide":[127],"variance":[128],"their":[130],"ratings":[131]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":3},{"year":2019,"cited_by_count":1},{"year":2018,"cited_by_count":1},{"year":2017,"cited_by_count":1},{"year":2016,"cited_by_count":2},{"year":2015,"cited_by_count":3},{"year":2014,"cited_by_count":1},{"year":2012,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}