{"id":"https://openalex.org/W2079713869","doi":"https://doi.org/10.1145/2669557.2669562","title":"Utility evaluation of models","display_name":"Utility evaluation of models","publication_year":2014,"publication_date":"2014-10-28","ids":{"openalex":"https://openalex.org/W2079713869","doi":"https://doi.org/10.1145/2669557.2669562","mag":"2079713869"},"language":"en","primary_location":{"id":"doi:10.1145/2669557.2669562","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2669557.2669562","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Fifth Workshop on Beyond Time and Errors: Novel Evaluation Methods for Visualization","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5059655101","display_name":"Jean Scholtz","orcid":null},"institutions":[{"id":"https://openalex.org/I142606810","display_name":"Pacific Northwest National Laboratory","ror":"https://ror.org/05h992307","country_code":"US","type":"facility","lineage":["https://openalex.org/I1325736334","https://openalex.org/I1330989302","https://openalex.org/I142606810","https://openalex.org/I39565521"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Jean Scholtz","raw_affiliation_strings":["Pacific Northwest National Laboratory, Richland, WA","Pacific Northwest National Laboratory, Richland, WA#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Pacific Northwest National Laboratory, Richland, WA","institution_ids":["https://openalex.org/I142606810"]},{"raw_affiliation_string":"Pacific Northwest National Laboratory, Richland, WA#TAB#","institution_ids":["https://openalex.org/I142606810"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077090795","display_name":"Oriana Love","orcid":null},"institutions":[{"id":"https://openalex.org/I142606810","display_name":"Pacific Northwest National Laboratory","ror":"https://ror.org/05h992307","country_code":"US","type":"facility","lineage":["https://openalex.org/I1325736334","https://openalex.org/I1330989302","https://openalex.org/I142606810","https://openalex.org/I39565521"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Oriana Love","raw_affiliation_strings":["Pacific Northwest National Laboratory, Richland, WA","Pacific Northwest National Laboratory, Richland, WA#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Pacific Northwest National Laboratory, Richland, WA","institution_ids":["https://openalex.org/I142606810"]},{"raw_affiliation_string":"Pacific Northwest National Laboratory, Richland, WA#TAB#","institution_ids":["https://openalex.org/I142606810"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034806125","display_name":"Mark Whiting","orcid":"https://orcid.org/0000-0002-4056-6784"},"institutions":[{"id":"https://openalex.org/I142606810","display_name":"Pacific Northwest National Laboratory","ror":"https://ror.org/05h992307","country_code":"US","type":"facility","lineage":["https://openalex.org/I1325736334","https://openalex.org/I1330989302","https://openalex.org/I142606810","https://openalex.org/I39565521"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mark Whiting","raw_affiliation_strings":["Pacific Northwest National Laboratory, Richland, WA","Pacific Northwest National Laboratory, Richland, WA#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Pacific Northwest National Laboratory, Richland, WA","institution_ids":["https://openalex.org/I142606810"]},{"raw_affiliation_string":"Pacific Northwest National Laboratory, Richland, WA#TAB#","institution_ids":["https://openalex.org/I142606810"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031877196","display_name":"Duncan Hodges","orcid":"https://orcid.org/0000-0002-0660-8776"},"institutions":[{"id":"https://openalex.org/I40120149","display_name":"University of Oxford","ror":"https://ror.org/052gg0110","country_code":"GB","type":"education","lineage":["https://openalex.org/I40120149"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Duncan Hodges","raw_affiliation_strings":["Oxford University Wolfson Building, Oxford, UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Oxford University Wolfson Building, Oxford, UK","institution_ids":["https://openalex.org/I40120149"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085656839","display_name":"Lia Emanuel","orcid":null},"institutions":[{"id":"https://openalex.org/I51601045","display_name":"University of Bath","ror":"https://ror.org/002h8g185","country_code":"GB","type":"education","lineage":["https://openalex.org/I51601045"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Lia Emanuel","raw_affiliation_strings":["University of Bath, Bath, NE Somerset, UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Bath, Bath, NE Somerset, UK","institution_ids":["https://openalex.org/I51601045"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5073105508","display_name":"Dana\u00eb Stanton Fraser","orcid":"https://orcid.org/0000-0002-3062-731X"},"institutions":[{"id":"https://openalex.org/I51601045","display_name":"University of Bath","ror":"https://ror.org/002h8g185","country_code":"GB","type":"education","lineage":["https://openalex.org/I51601045"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Dana\u00eb Stanton Fraser","raw_affiliation_strings":["University of Bath, Bath, NE Somerset, UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Bath, Bath, NE Somerset, UK","institution_ids":["https://openalex.org/I51601045"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5059655101"],"corresponding_institution_ids":["https://openalex.org/I142606810"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.14008029,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":94},"biblio":{"volume":null,"issue":null,"first_page":"160","last_page":"167"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10799","display_name":"Data Visualization and Analytics","score":0.9936000108718872,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10799","display_name":"Data Visualization and Analytics","score":0.9936000108718872,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10470","display_name":"Usability and User Interface Design","score":0.974399983882904,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14074","display_name":"Persona Design and Applications","score":0.9722999930381775,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7531559467315674},{"id":"https://openalex.org/keywords/identification","display_name":"Identification (biology)","score":0.7030853629112244},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.48319900035858154},{"id":"https://openalex.org/keywords/work","display_name":"Work (physics)","score":0.4512542486190796},{"id":"https://openalex.org/keywords/management-science","display_name":"Management science","score":0.4485251009464264},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.4356168508529663},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.4028719663619995},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.17336416244506836}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7531559467315674},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.7030853629112244},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.48319900035858154},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.4512542486190796},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.4485251009464264},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4356168508529663},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.4028719663619995},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.17336416244506836},{"id":"https://openalex.org/C59822182","wikidata":"https://www.wikidata.org/wiki/Q441","display_name":"Botany","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/2669557.2669562","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2669557.2669562","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Fifth Workshop on Beyond Time and Errors: Novel Evaluation Methods for Visualization","raw_type":"proceedings-article"},{"id":"pmh:oai:opus.bath.ac.uk:42074","is_oa":false,"landing_page_url":"http://opus.bath.ac.uk/42074/1/Scholtz_et_al._BELIV_2014_Utility_evaluation_of_models_pre_print_.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306401631","display_name":"The University of Bath Online Publications Store (The University of Bath)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I51601045","host_organization_name":"University of Bath","host_organization_lineage":["https://openalex.org/I51601045"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Book Sections"},{"id":"pmh:oai:purehost.bath.ac.uk:publications/aeb49ef2-7a14-4c4d-a140-19f6547f5b67","is_oa":false,"landing_page_url":"https://researchportal.bath.ac.uk/en/publications/aeb49ef2-7a14-4c4d-a140-19f6547f5b67","pdf_url":null,"source":{"id":"https://openalex.org/S4377196294","display_name":"Pure (University of Bath)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I51601045","host_organization_name":"University of Bath","host_organization_lineage":["https://openalex.org/I51601045"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":""}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1899721271","display_name":null,"funder_award_id":"AC05-76RL01830","funder_id":"https://openalex.org/F4320306250","funder_display_name":"Battelle"},{"id":"https://openalex.org/G2506945562","display_name":null,"funder_award_id":"76RL01830","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G2510423872","display_name":null,"funder_award_id":"AC05-76RL01830","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G3196714330","display_name":null,"funder_award_id":"DE-AC05-76RL01830","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G3448260292","display_name":null,"funder_award_id":"Contract DE-AC05-76RL01830","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G3887989202","display_name":null,"funder_award_id":"EP/J004995/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G5200347334","display_name":null,"funder_award_id":"DE-AC05-76RL01830","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G5321382714","display_name":null,"funder_award_id":"76RL01830","funder_id":"https://openalex.org/F4320338354","funder_display_name":"Pacific Northwest National Laboratory"},{"id":"https://openalex.org/G5726405315","display_name":null,"funder_award_id":"DE-AC05","funder_id":"https://openalex.org/F4320306250","funder_display_name":"Battelle"},{"id":"https://openalex.org/G6664334341","display_name":null,"funder_award_id":"DE-AC05-76RL01830","funder_id":"https://openalex.org/F4320306250","funder_display_name":"Battelle"},{"id":"https://openalex.org/G7334037170","display_name":"SID: An Exploration of Super-Identity","funder_award_id":"EP/J004995/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G7995982022","display_name":null,"funder_award_id":"DE-AC05","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"}],"funders":[{"id":"https://openalex.org/F4320306084","display_name":"U.S. Department of Energy","ror":"https://ror.org/01bj3aw27"},{"id":"https://openalex.org/F4320306110","display_name":"U.S. Department of Homeland Security","ror":"https://ror.org/00jyr0d86"},{"id":"https://openalex.org/F4320306250","display_name":"Battelle","ror":"https://ror.org/01h5tnr73"},{"id":"https://openalex.org/F4320332178","display_name":"National Institute of Standards and Technology","ror":"https://ror.org/05xpvk416"},{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"},{"id":"https://openalex.org/F4320338354","display_name":"Pacific Northwest National Laboratory","ror":"https://ror.org/05h992307"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":11,"referenced_works":["https://openalex.org/W199261454","https://openalex.org/W1967134698","https://openalex.org/W1991836334","https://openalex.org/W2072162563","https://openalex.org/W2092795326","https://openalex.org/W2106818893","https://openalex.org/W2117393561","https://openalex.org/W2121555553","https://openalex.org/W2142493242","https://openalex.org/W2164349448","https://openalex.org/W2167439408"],"related_works":["https://openalex.org/W2898732673","https://openalex.org/W2410053581","https://openalex.org/W2383658677","https://openalex.org/W3123203398","https://openalex.org/W1972473893","https://openalex.org/W2466435674","https://openalex.org/W2765200542","https://openalex.org/W2367893528","https://openalex.org/W3107784576","https://openalex.org/W242750434"],"abstract_inverted_index":{"In":[0],"this":[1],"paper,":[2],"we":[3,64],"present":[4,86],"three":[5,38],"case":[6],"studies":[7],"of":[8,11,36],"utility":[9],"evaluations":[10],"underlying":[12],"models":[13,22],"in":[14,26,54,69],"software":[15],"systems:":[16],"a":[17,29,41],"user-model,":[18],"technical":[19],"and":[20,25,28,48,56,66,85],"social":[21],"both":[23],"singly":[24],"combination,":[27],"research-based":[30],"model":[31,47],"for":[32,80,88],"user":[33],"identification.":[34],"Each":[35],"the":[37,46,58,62,71,75],"cases":[39],"used":[40,65],"different":[42],"approach":[43],"to":[44,52],"evaluating":[45],"each":[49],"had":[50],"challenges":[51,67],"overcome":[53],"designing":[55,70],"implementing":[57],"evaluation.":[59],"We":[60],"describe":[61],"methods":[63],"faced":[68],"evaluation":[72],"procedures,":[73],"summarize":[74],"lessons":[76],"learned,":[77],"enumerate":[78],"considerations":[79],"those":[81],"undertaking":[82],"such":[83],"evaluations,":[84],"directions":[87],"future":[89],"work.":[90]},"counts_by_year":[{"year":2024,"cited_by_count":1}],"updated_date":"2026-05-04T08:30:34.212998","created_date":"2025-10-10T00:00:00"}