{"id":"https://openalex.org/W4200523759","doi":"https://doi.org/10.31234/osf.io/j8t3c","title":"Requirements for the Empirical Assessment of Human-AI Work Systems: A Contribution to AI Measurement Science","display_name":"Requirements for the Empirical Assessment of Human-AI Work Systems: A Contribution to AI Measurement Science","publication_year":2021,"publication_date":"2021-11-16","ids":{"openalex":"https://openalex.org/W4200523759","doi":"https://doi.org/10.31234/osf.io/j8t3c"},"language":"en","primary_location":{"id":"doi:10.31234/osf.io/j8t3c","doi":"https://doi.org/10.31234/osf.io/j8t3c","is_oa":false,"landing_page_url":"https://doi.org/10.31234/osf.io/j8t3c","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"},"volume":null,"type":"preprint","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Gary Klein","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gary Klein","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Robert Hoffman","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Robert Hoffman","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086667759","display_name":"Shane T. Mueller","orcid":"https://orcid.org/0000-0001-7418-4592","observed_orcids":["https://orcid.org/0000-0001-7418-4592"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shane T. Mueller","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0001-7418-4592","affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5048181212","display_name":"William J. Clancey","orcid":"https://orcid.org/0000-0001-9643-9437","observed_orcids":["https://orcid.org/0000-0001-9643-9437"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"William Clancey","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10525","display_name":"Human-Automation Interaction and Safety","score":0.5205000042915344,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10525","display_name":"Human-Automation Interaction and Safety","score":0.5205000042915344,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10470","display_name":"Usability and User Interface Design","score":0.364300012588501,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11024","display_name":"Information Systems Theories and Implementation","score":0.01119999960064888,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ai-evaluation","display_name":"AI evaluation","score":0.8140000104904175},{"id":"https://openalex.org/keywords/study-design","display_name":"research design","score":0.7919999957084656},{"id":"https://openalex.org/keywords/online-experiments","display_name":"online experiments","score":0.7429999709129333},{"id":"https://openalex.org/keywords/statistical-analysis","display_name":"statistical analysis","score":0.6480000019073486},{"id":"https://openalex.org/keywords/learnability","display_name":"learnability","score":0.5479999780654907},{"id":"https://openalex.org/keywords/usability","display_name":"usability","score":0.949999988079071}],"concepts":[{"id":"https://openalex.org/C2780615836","wikidata":"https://www.wikidata.org/wiki/Q2471869","display_name":"USable","level":2,"score":0.6566016674041748},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6512119174003601},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.631987452507019},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.49942994117736816},{"id":"https://openalex.org/C33566652","wikidata":"https://www.wikidata.org/wiki/Q1065927","display_name":"Theme (computing)","level":2,"score":0.45357218384742737},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4117441177368164},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.39633098244667053},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.38095617294311523},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.380397230386734},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3548741936683655},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.34046250581741333},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.2457098364830017},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.31234/osf.io/j8t3c","doi":"https://doi.org/10.31234/osf.io/j8t3c","is_oa":false,"landing_page_url":"https://doi.org/10.31234/osf.io/j8t3c","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"},{"id":"pmh:oai:share.osf.io:8fbfb501-8a36-4a67-91dc-5b9f095f0114","doi":null,"is_oa":false,"landing_page_url":"https://osf.io/j8t3c","pdf_url":null,"source":{"id":"https://openalex.org/S4306401687","display_name":"PsyArXiv (OSF Preprints)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2799848540","host_organization_name":"Center for Open Science","host_organization_lineage":["https://openalex.org/I2799848540"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"pmh:oai:share.osf.io:E0018-0CA-345","doi":null,"is_oa":false,"landing_page_url":"http://doi.org/10.31234/OSF.IO/J8T3C","pdf_url":null,"source":{"id":"https://openalex.org/S4306401687","display_name":"PsyArXiv (OSF Preprints)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2799848540","host_organization_name":"Center for Open Science","host_organization_lineage":["https://openalex.org/I2799848540"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"preprint"}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4200284456","https://openalex.org/W4385635581","https://openalex.org/W2084756473","https://openalex.org/W3215890839","https://openalex.org/W7221691542","https://openalex.org/W7203602634","https://openalex.org/W4401514137","https://openalex.org/W7162699314","https://openalex.org/W4379745096","https://openalex.org/W4388802931"],"abstract_inverted_index":{"The":[0,45,134],"development":[1],"of":[2,14,23,66,69,77,95,110,152],"AI":[3,38,54,71,117,153],"systems":[4,26],"represents":[5],"a":[6,67,108],"significant":[7],"investment.":[8],"But":[9],"to":[10,47,116,144,148],"realize":[11],"the":[12,33,51,83,96,150],"promise":[13],"that":[15,32,53,86,98,113],"investment,":[16],"performance":[17],"assessment":[18,55],"is":[19,50,107],"necessary.":[20],"Empirical":[21,120],"evaluation":[22,118],"Human-AI":[24],"work":[25,34],"must":[27,56,60],"adduce":[28],"convincing":[29],"empirical":[30,84],"evidence":[31],"method":[35],"and":[36,43,131],"its":[37],"technology":[39],"are":[40,87,99,114,122],"learnable,":[41],"usable,":[42],"useful.":[44],"theme":[46],"this":[48,90,139],"Report":[49,91,140],"notion":[52],"be":[57,62,142],"effective":[58],"but":[59],"also":[61],"efficient.":[63],"Bench":[64],"testing":[65],"prototype":[68],"an":[70],"system":[72],"cannot":[73],"require":[74],"extensive":[75],"series":[76],"experiments":[78],"with":[79],"complex":[80],"designs.":[81],"Thus,":[82],"requirements":[85,121,136],"presented":[88,123,137],"in":[89,101,138],"involve":[92],"escaping":[93],"some":[94],"constraints":[97,112],"imposed":[100],"traditional":[102],"laboratory":[103],"research.":[104],"Also,":[105],"there":[106],"recognition":[109],"new":[111],"unique":[115],"contexts.":[119],"covering":[124],"study":[125],"design,":[126],"research":[127,146],"methods,":[128],"statistical":[129],"analyses,":[130],"online":[132],"experimentation.":[133],"15":[135],"should":[141],"applicable":[143],"all":[145],"intended":[147],"evaluate":[149],"effectivity":[151],"systems.":[154]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2023,"cited_by_count":1}],"updated_date":"2026-09-29T14:09:16.559479","created_date":"2021-12-31T00:00:00"}