{"id":"https://openalex.org/W3004390862","doi":"https://doi.org/10.1145/3046947","title":"Assessing Human Error Against a Benchmark of Perfection","display_name":"Assessing Human Error Against a Benchmark of Perfection","publication_year":2017,"publication_date":"2017-07-27","ids":{"openalex":"https://openalex.org/W3004390862","doi":"https://doi.org/10.1145/3046947","mag":"3004390862"},"language":"en","primary_location":{"id":"doi:10.1145/3046947","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3046947","pdf_url":null,"source":{"id":"https://openalex.org/S41523882","display_name":"ACM Transactions on Knowledge Discovery from Data","issn_l":"1556-4681","issn":["1556-4681","1556-472X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Knowledge Discovery from Data","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5048789742","display_name":"Ashton Anderson","orcid":"https://orcid.org/0000-0003-3089-6883"},"institutions":[{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Ashton Anderson","raw_affiliation_strings":["Microsoft Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research","institution_ids":["https://openalex.org/I4210164937"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055710645","display_name":"Jon Kleinberg","orcid":"https://orcid.org/0000-0002-1929-2512"},"institutions":[{"id":"https://openalex.org/I205783295","display_name":"Cornell University","ror":"https://ror.org/05bnh6r87","country_code":"US","type":"education","lineage":["https://openalex.org/I205783295"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jon Kleinberg","raw_affiliation_strings":["Cornell University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Cornell University","institution_ids":["https://openalex.org/I205783295"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5034703281","display_name":"Sendhil Mullainathan","orcid":"https://orcid.org/0000-0001-8508-4052"},"institutions":[{"id":"https://openalex.org/I2801851002","display_name":"Harvard University Press","ror":"https://ror.org/006v7bf86","country_code":"US","type":"other","lineage":["https://openalex.org/I136199984","https://openalex.org/I2801851002"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sendhil Mullainathan","raw_affiliation_strings":["Harvard University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Harvard University","institution_ids":["https://openalex.org/I2801851002"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.5703,"has_fulltext":false,"cited_by_count":21,"citation_normalized_percentile":{"value":0.67496444,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":"11","issue":"4","first_page":"1","last_page":"25"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10315","display_name":"Decision-Making and Behavioral Economics","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1800","display_name":"General Decision Sciences"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10315","display_name":"Decision-Making and Behavioral Economics","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1800","display_name":"General Decision Sciences"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10042","display_name":"Neural and Behavioral Psychology Studies","score":0.993399977684021,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T11674","display_name":"Sports Analytics and Performance","score":0.9894000291824341,"subfield":{"id":"https://openalex.org/subfields/2002","display_name":"Economics and Econometrics"},"field":{"id":"https://openalex.org/fields/20","display_name":"Economics, Econometrics and Finance"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7530028820037842},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5977328419685364},{"id":"https://openalex.org/keywords/trace","display_name":"TRACE (psycholinguistics)","score":0.5901491641998291},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5454412698745728},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.5370650887489319},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.5368673205375671},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.535811722278595},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5144091248512268},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.4886951744556427},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.42592653632164},{"id":"https://openalex.org/keywords/domain-knowledge","display_name":"Domain knowledge","score":0.4204397201538086},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.12324127554893494}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7530028820037842},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5977328419685364},{"id":"https://openalex.org/C75291252","wikidata":"https://www.wikidata.org/wiki/Q1315756","display_name":"TRACE (psycholinguistics)","level":2,"score":0.5901491641998291},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5454412698745728},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.5370650887489319},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.5368673205375671},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.535811722278595},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5144091248512268},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4886951744556427},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.42592653632164},{"id":"https://openalex.org/C207685749","wikidata":"https://www.wikidata.org/wiki/Q2088941","display_name":"Domain knowledge","level":2,"score":0.4204397201538086},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.12324127554893494},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3046947","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3046947","pdf_url":null,"source":{"id":"https://openalex.org/S41523882","display_name":"ACM Transactions on Knowledge Discovery from Data","issn_l":"1556-4681","issn":["1556-4681","1556-472X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Knowledge Discovery from Data","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.8199999928474426}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":21,"referenced_works":["https://openalex.org/W158727920","https://openalex.org/W305062067","https://openalex.org/W1546453605","https://openalex.org/W1945711486","https://openalex.org/W1974964983","https://openalex.org/W1991751824","https://openalex.org/W1998711856","https://openalex.org/W2005237197","https://openalex.org/W2033797867","https://openalex.org/W2048493867","https://openalex.org/W2077833524","https://openalex.org/W2088381906","https://openalex.org/W2091785129","https://openalex.org/W2103278175","https://openalex.org/W2120197817","https://openalex.org/W2141630370","https://openalex.org/W2159661094","https://openalex.org/W2293709543","https://openalex.org/W2396991733","https://openalex.org/W2502334226","https://openalex.org/W4239427658"],"related_works":["https://openalex.org/W2378211422","https://openalex.org/W2745001401","https://openalex.org/W4321353415","https://openalex.org/W2130974462","https://openalex.org/W972276598","https://openalex.org/W1583422155","https://openalex.org/W2158321484","https://openalex.org/W3213252596","https://openalex.org/W1649619740","https://openalex.org/W1534006406"],"abstract_inverted_index":{"An":[0],"increasing":[1],"number":[2],"of":[3,23,36,50,123,129,157,170,177,191,201,204,219],"domains":[4],"are":[5,54,162,222],"providing":[6],"us":[7],"with":[8,78,110],"detailed":[9],"trace":[10],"data":[11],"on":[12,74,229],"human":[13,66,171],"decisions":[14,25,51,87],"in":[15,82,96,147,164,198,211],"settings":[16,166],"where":[17,52,167],"we":[18,43,72,160],"can":[19,44],"evaluate":[20],"the":[21,48,83,86,144,148,168,175,178,180,185,188,192,216],"quality":[22],"these":[24,202],"via":[26],"an":[27,33,220],"algorithm.":[28],"Motivated":[29],"by":[30,89,137],"this":[31],"development,":[32],"emerging":[34],"line":[35],"work":[37],"has":[38],"begun":[39],"to":[40,56,119,183],"consider":[41],"whether":[42],"characterize":[45],"and":[46,115,187,206],"predict":[47],"kinds":[49],"people":[53],"likely":[55],"make":[57,184],"errors.":[58],"To":[59],"investigate":[60],"what":[61],"a":[62,75,79,97,105,121,127],"general":[63],"framework":[64],"for":[65,126,142],"error":[67,172],"prediction":[68],"might":[69],"look":[70],"like,":[71],"focus":[73],"model":[76],"system":[77],"rich":[80,196],"history":[81],"behavioral":[84],"sciences:":[85],"made":[88],"chess":[90,117,130],"players":[91,146],"as":[92],"they":[93],"select":[94],"moves":[95],"game.":[98],"We":[99,150,194],"carry":[100],"out":[101],"our":[102,152,212],"analysis":[103,153,169],"at":[104],"large":[106],"scale,":[107],"employing":[108],"datasets":[109],"several":[111],"million":[112],"recorded":[113],"games,":[114],"using":[116],"tablebases":[118],"acquire":[120],"form":[122],"ground":[124],"truth":[125],"subset":[128],"positions":[131],"that":[132,159,210],"have":[133],"been":[134],"completely":[135],"solved":[136],"computers":[138],"but":[139],"remain":[140],"challenging":[141],"even":[143],"best":[145],"world.":[149],"organize":[151],"around":[154],"three":[155,200],"categories":[156,203],"features":[158,214,227],"argue":[161],"present":[163],"most":[165],"is":[173],"applicable:":[174],"skill":[176,230],"decision-maker,":[179],"time":[181],"available":[182],"decision,":[186],"inherent":[189,217],"difficulty":[190,218],"decision.":[193],"identify":[195],"structure":[197],"all":[199],"features,":[205],"find":[207],"strong":[208],"evidence":[209],"domain,":[213],"describing":[215],"instance":[221],"significantly":[223],"more":[224],"powerful":[225],"than":[226],"based":[228],"or":[231],"time.":[232]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":1},{"year":2017,"cited_by_count":1}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}