{"id":"https://openalex.org/W2113547941","doi":"https://doi.org/10.1177/1473871613490290","title":"Evaluation of visual analytics environments: The road to the Visual Analytics Science and Technology challenge evaluation methodology","display_name":"Evaluation of visual analytics environments: The road to the Visual Analytics Science and Technology challenge evaluation methodology","publication_year":2013,"publication_date":"2013-06-11","ids":{"openalex":"https://openalex.org/W2113547941","doi":"https://doi.org/10.1177/1473871613490290","mag":"2113547941"},"language":"en","primary_location":{"id":"doi:10.1177/1473871613490290","is_oa":false,"landing_page_url":"https://doi.org/10.1177/1473871613490290","pdf_url":null,"source":{"id":"https://openalex.org/S55152591","display_name":"Information Visualization","issn_l":"1473-8716","issn":["1473-8716","1473-8724"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320017","host_organization_name":"SAGE Publishing","host_organization_lineage":["https://openalex.org/P4310320017"],"host_organization_lineage_names":["SAGE Publishing"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information Visualization","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5059655101","display_name":"Jean Scholtz","orcid":null},"institutions":[{"id":"https://openalex.org/I142606810","display_name":"Pacific Northwest National Laboratory","ror":"https://ror.org/05h992307","country_code":"US","type":"facility","lineage":["https://openalex.org/I1325736334","https://openalex.org/I1330989302","https://openalex.org/I142606810","https://openalex.org/I39565521"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Jean Scholtz","raw_affiliation_strings":["Pacific Northwest National Laboratory, Richland, WA, USA","Pacific Northwest National Laboratory, Richland, WA USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Pacific Northwest National Laboratory, Richland, WA, USA","institution_ids":["https://openalex.org/I142606810"]},{"raw_affiliation_string":"Pacific Northwest National Laboratory, Richland, WA USA","institution_ids":["https://openalex.org/I142606810"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030084361","display_name":"Catherine Plaisant","orcid":"https://orcid.org/0000-0003-4049-5848"},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Catherine Plaisant","raw_affiliation_strings":["University of Maryland, College Park, MD, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Maryland, College Park, MD, USA","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034806125","display_name":"Mark Whiting","orcid":"https://orcid.org/0000-0002-4056-6784"},"institutions":[{"id":"https://openalex.org/I142606810","display_name":"Pacific Northwest National Laboratory","ror":"https://ror.org/05h992307","country_code":"US","type":"facility","lineage":["https://openalex.org/I1325736334","https://openalex.org/I1330989302","https://openalex.org/I142606810","https://openalex.org/I39565521"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mark Whiting","raw_affiliation_strings":["Pacific Northwest National Laboratory, Richland, WA, USA","Pacific Northwest National Laboratory, Richland, WA USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Pacific Northwest National Laboratory, Richland, WA, USA","institution_ids":["https://openalex.org/I142606810"]},{"raw_affiliation_string":"Pacific Northwest National Laboratory, Richland, WA USA","institution_ids":["https://openalex.org/I142606810"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5021770726","display_name":"Georges Grinstein","orcid":"https://orcid.org/0000-0003-2388-6601"},"institutions":[{"id":"https://openalex.org/I133738476","display_name":"University of Massachusetts Lowell","ror":"https://ror.org/03hamhx47","country_code":"US","type":"education","lineage":["https://openalex.org/I133738476"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Georges Grinstein","raw_affiliation_strings":["University of Massachusetts Lowell, Lowell, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Massachusetts Lowell, Lowell, MA, USA","institution_ids":["https://openalex.org/I133738476"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5059655101"],"corresponding_institution_ids":["https://openalex.org/I142606810"],"apc_list":null,"apc_paid":null,"fwci":1.3809,"has_fulltext":false,"cited_by_count":25,"citation_normalized_percentile":{"value":0.84650359,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":"13","issue":"4","first_page":"326","last_page":"335"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10799","display_name":"Data Visualization and Analytics","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10799","display_name":"Data Visualization and Analytics","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8405807018280029},{"id":"https://openalex.org/keywords/visual-analytics","display_name":"Visual analytics","score":0.7763267755508423},{"id":"https://openalex.org/keywords/analytics","display_name":"Analytics","score":0.6898565888404846},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.6476826667785645},{"id":"https://openalex.org/keywords/cultural-analytics","display_name":"Cultural analytics","score":0.5856555104255676},{"id":"https://openalex.org/keywords/resource","display_name":"Resource (disambiguation)","score":0.5744463205337524},{"id":"https://openalex.org/keywords/pace","display_name":"Pace","score":0.5643610954284668},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.5606896281242371},{"id":"https://openalex.org/keywords/interactive-visual-analysis","display_name":"Interactive visual analysis","score":0.522409200668335},{"id":"https://openalex.org/keywords/software-analytics","display_name":"Software analytics","score":0.42048734426498413},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.40695565938949585},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.2521306276321411},{"id":"https://openalex.org/keywords/semantic-analytics","display_name":"Semantic analytics","score":0.2442161738872528},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.2312374711036682},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.16413614153862},{"id":"https://openalex.org/keywords/the-internet","display_name":"The Internet","score":0.12669628858566284},{"id":"https://openalex.org/keywords/software-development-process","display_name":"Software development process","score":0.09374085068702698}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8405807018280029},{"id":"https://openalex.org/C59732488","wikidata":"https://www.wikidata.org/wiki/Q2528440","display_name":"Visual analytics","level":3,"score":0.7763267755508423},{"id":"https://openalex.org/C79158427","wikidata":"https://www.wikidata.org/wiki/Q485396","display_name":"Analytics","level":2,"score":0.6898565888404846},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.6476826667785645},{"id":"https://openalex.org/C545860419","wikidata":"https://www.wikidata.org/wiki/Q5193251","display_name":"Cultural analytics","level":5,"score":0.5856555104255676},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.5744463205337524},{"id":"https://openalex.org/C2777526511","wikidata":"https://www.wikidata.org/wiki/Q691543","display_name":"Pace","level":2,"score":0.5643610954284668},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.5606896281242371},{"id":"https://openalex.org/C99740376","wikidata":"https://www.wikidata.org/wiki/Q17092520","display_name":"Interactive visual analysis","level":4,"score":0.522409200668335},{"id":"https://openalex.org/C171981572","wikidata":"https://www.wikidata.org/wiki/Q7554239","display_name":"Software analytics","level":5,"score":0.42048734426498413},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.40695565938949585},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.2521306276321411},{"id":"https://openalex.org/C148792806","wikidata":"https://www.wikidata.org/wiki/Q7449046","display_name":"Semantic analytics","level":4,"score":0.2442161738872528},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.2312374711036682},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.16413614153862},{"id":"https://openalex.org/C110875604","wikidata":"https://www.wikidata.org/wiki/Q75","display_name":"The Internet","level":2,"score":0.12669628858566284},{"id":"https://openalex.org/C180152950","wikidata":"https://www.wikidata.org/wiki/Q2904257","display_name":"Software development process","level":4,"score":0.09374085068702698},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C130436687","wikidata":"https://www.wikidata.org/wiki/Q7978591","display_name":"Web modeling","level":3,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1177/1473871613490290","is_oa":false,"landing_page_url":"https://doi.org/10.1177/1473871613490290","pdf_url":null,"source":{"id":"https://openalex.org/S55152591","display_name":"Information Visualization","issn_l":"1473-8716","issn":["1473-8716","1473-8724"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320017","host_organization_name":"SAGE Publishing","host_organization_lineage":["https://openalex.org/P4310320017"],"host_organization_lineage_names":["SAGE Publishing"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information Visualization","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":24,"referenced_works":["https://openalex.org/W52809394","https://openalex.org/W1490854657","https://openalex.org/W1500842519","https://openalex.org/W1540101265","https://openalex.org/W1594453896","https://openalex.org/W1602715654","https://openalex.org/W1981355240","https://openalex.org/W2000143631","https://openalex.org/W2018850751","https://openalex.org/W2057615322","https://openalex.org/W2058203255","https://openalex.org/W2119895915","https://openalex.org/W2133732523","https://openalex.org/W2142493242","https://openalex.org/W2144320230","https://openalex.org/W2150485981","https://openalex.org/W2158685692","https://openalex.org/W2161676054","https://openalex.org/W2166699593","https://openalex.org/W2167439408","https://openalex.org/W2168440500","https://openalex.org/W2346085410","https://openalex.org/W2506550795","https://openalex.org/W2966704589"],"related_works":["https://openalex.org/W2062940763","https://openalex.org/W2167937298","https://openalex.org/W2153940791","https://openalex.org/W2018276586","https://openalex.org/W2116412677","https://openalex.org/W2518957967","https://openalex.org/W2744288354","https://openalex.org/W3023517071","https://openalex.org/W2181391231","https://openalex.org/W2229630036"],"abstract_inverted_index":{"Evaluation":[0],"of":[1,29,32,62,134,157,171],"software":[2],"can":[3],"take":[4],"many":[5,141,221],"forms":[6],"ranging":[7],"from":[8,96],"algorithm":[9],"correctness":[10],"and":[11,44,60,66,74,76,83,99,113,128,167,176,189,211],"performance":[12],"to":[13,20,58,69,94,100,153,165,185,190,220],"evaluations":[14],"that":[15,192],"focus":[16,133],"on":[17,206,214],"the":[18,21,30,78,81,84,92,110,117,124,132,151,155,158,162,168,187,193,200,207],"value":[19],"end":[22],"user.":[23],"This":[24,54],"article":[25],"presents":[26],"a":[27,50,146],"discussion":[28],"development":[31],"an":[33],"evaluation":[34,52,148,194],"methodology":[35],"for":[36,140],"visual":[37,63],"analytics":[38,64],"environments.":[39],"The":[40,204],"Visual":[41,125,208],"Analytics":[42,126,209],"Science":[43,127,210],"Technology":[45,129,212],"Challenge":[46,130,213],"was":[47],"created":[48],"as":[49],"community":[51,93,147,224],"resource.":[53],"resource":[55],"is":[56,131,182],"available":[57],"researchers":[59],"developers":[61],"environments":[65],"allows":[67,91],"them":[68],"test":[70],"out":[71],"their":[72],"designs":[73],"visualization":[75],"compare":[77],"results":[79,90],"with":[80,199],"solution":[82],"entries":[85],"prepared":[86],"by":[87],"others.":[88],"Sharing":[89],"learn":[95],"each":[97],"other":[98],"hopefully":[101],"advance":[102],"more":[103],"quickly.":[104],"In":[105],"this":[106,135],"article,":[107,136],"we":[108],"discuss":[109],"original":[111],"challenge":[112],"its":[114,121],"evolution":[115,188],"during":[116],"7":[118],"years":[119],"since":[120],"inception.":[122],"While":[123],"there":[137],"are":[138,196],"lessons":[139],"involved":[142],"in":[143,223],"setting":[144],"up":[145],"program,":[149],"including":[150,174],"need":[152],"understand":[154],"purpose":[156],"evaluation,":[159],"decide":[160],"upon":[161],"right":[163],"metrics":[164,173],"use,":[166],"appropriate":[169],"implementation":[170],"those":[172],"datasets":[175],"evaluators.":[177],"For":[178],"ongoing":[179],"evaluations,":[180],"it":[181],"also":[183],"necessary":[184],"track":[186],"ensure":[191],"methodologies":[195],"keeping":[197],"pace":[198],"science":[201],"being":[202],"evaluated.":[203],"discussions":[205],"these":[215],"topics":[216],"should":[217],"be":[218],"pertinent":[219],"interested":[222],"evaluations.":[225]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":3},{"year":2020,"cited_by_count":3},{"year":2019,"cited_by_count":4},{"year":2018,"cited_by_count":5},{"year":2017,"cited_by_count":2},{"year":2015,"cited_by_count":3},{"year":2014,"cited_by_count":2}],"updated_date":"2026-05-04T06:00:05.808912","created_date":"2025-10-10T00:00:00"}