{"id":"https://openalex.org/W4387885672","doi":"https://doi.org/10.1109/tvcg.2023.3326591","title":"Are We Closing the Loop Yet? Gaps in the Generalizability of VIS4ML Research","display_name":"Are We Closing the Loop Yet? Gaps in the Generalizability of VIS4ML Research","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4387885672","doi":"https://doi.org/10.1109/tvcg.2023.3326591","pmid":"https://pubmed.ncbi.nlm.nih.gov/37871059"},"language":"en","primary_location":{"id":"doi:10.1109/tvcg.2023.3326591","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tvcg.2023.3326591","pdf_url":null,"source":{"id":"https://openalex.org/S84775595","display_name":"IEEE Transactions on Visualization and Computer Graphics","issn_l":"1077-2626","issn":["1077-2626","1941-0506","2160-9306"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Visualization and Computer Graphics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5111056004","display_name":"Hariharan Subramonyam","orcid":null},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Hariharan Subramonyam","raw_affiliation_strings":["Stanford University, USA"],"affiliations":[{"raw_affiliation_string":"Stanford University, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5068008545","display_name":"Jessica Hullman","orcid":"https://orcid.org/0000-0001-6826-3550"},"institutions":[{"id":"https://openalex.org/I111979921","display_name":"Northwestern University","ror":"https://ror.org/000e0be47","country_code":"US","type":"education","lineage":["https://openalex.org/I111979921"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jessica Hullman","raw_affiliation_strings":["Northwestern University, USA"],"affiliations":[{"raw_affiliation_string":"Northwestern University, USA","institution_ids":["https://openalex.org/I111979921"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5111056004"],"corresponding_institution_ids":["https://openalex.org/I97018004"],"apc_list":null,"apc_paid":null,"fwci":1.1054,"has_fulltext":false,"cited_by_count":9,"citation_normalized_percentile":{"value":0.80232481,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":"30","issue":"1","first_page":"1","last_page":"11"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10799","display_name":"Data Visualization and Analytics","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10799","display_name":"Data Visualization and Analytics","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13283","display_name":"Mental Health Research Topics","score":0.982200026512146,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9391000270843506,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/generalizability-theory","display_name":"Generalizability theory","score":0.8832394480705261},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8016847372055054},{"id":"https://openalex.org/keywords/generality","display_name":"Generality","score":0.7334740161895752},{"id":"https://openalex.org/keywords/human-in-the-loop","display_name":"Human-in-the-loop","score":0.6440845727920532},{"id":"https://openalex.org/keywords/documentation","display_name":"Documentation","score":0.5904056429862976},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5306372046470642},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.5126573443412781},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.503544270992279},{"id":"https://openalex.org/keywords/closing","display_name":"Closing (real estate)","score":0.4550907611846924},{"id":"https://openalex.org/keywords/scope","display_name":"Scope (computer science)","score":0.4288167953491211},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4055827856063843},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.37424227595329285},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.3653447926044464},{"id":"https://openalex.org/keywords/knowledge-management","display_name":"Knowledge management","score":0.35510456562042236},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.09688961505889893}],"concepts":[{"id":"https://openalex.org/C27158222","wikidata":"https://www.wikidata.org/wiki/Q5532422","display_name":"Generalizability theory","level":2,"score":0.8832394480705261},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8016847372055054},{"id":"https://openalex.org/C2780767217","wikidata":"https://www.wikidata.org/wiki/Q5532421","display_name":"Generality","level":2,"score":0.7334740161895752},{"id":"https://openalex.org/C2780626000","wikidata":"https://www.wikidata.org/wiki/Q5936775","display_name":"Human-in-the-loop","level":2,"score":0.6440845727920532},{"id":"https://openalex.org/C56666940","wikidata":"https://www.wikidata.org/wiki/Q788790","display_name":"Documentation","level":2,"score":0.5904056429862976},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5306372046470642},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5126573443412781},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.503544270992279},{"id":"https://openalex.org/C2778775528","wikidata":"https://www.wikidata.org/wiki/Q5135432","display_name":"Closing (real estate)","level":2,"score":0.4550907611846924},{"id":"https://openalex.org/C2778012447","wikidata":"https://www.wikidata.org/wiki/Q1034415","display_name":"Scope (computer science)","level":2,"score":0.4288167953491211},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4055827856063843},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.37424227595329285},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3653447926044464},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.35510456562042236},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.09688961505889893},{"id":"https://openalex.org/C138496976","wikidata":"https://www.wikidata.org/wiki/Q175002","display_name":"Developmental psychology","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C542102704","wikidata":"https://www.wikidata.org/wiki/Q183257","display_name":"Psychotherapist","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tvcg.2023.3326591","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tvcg.2023.3326591","pdf_url":null,"source":{"id":"https://openalex.org/S84775595","display_name":"IEEE Transactions on Visualization and Computer Graphics","issn_l":"1077-2626","issn":["1077-2626","1941-0506","2160-9306"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Visualization and Computer Graphics","raw_type":"journal-article"},{"id":"pmid:37871059","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/37871059","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on visualization and computer graphics","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":97,"referenced_works":["https://openalex.org/W57699484","https://openalex.org/W73593153","https://openalex.org/W1686810756","https://openalex.org/W1966087765","https://openalex.org/W1970569592","https://openalex.org/W1985898303","https://openalex.org/W1987135115","https://openalex.org/W1992743299","https://openalex.org/W2012457675","https://openalex.org/W2058203255","https://openalex.org/W2063534350","https://openalex.org/W2084776154","https://openalex.org/W2088290398","https://openalex.org/W2090687959","https://openalex.org/W2142493242","https://openalex.org/W2161078926","https://openalex.org/W2343061342","https://openalex.org/W2542768043","https://openalex.org/W2553981914","https://openalex.org/W2583408446","https://openalex.org/W2602814102","https://openalex.org/W2607223307","https://openalex.org/W2751298778","https://openalex.org/W2751722336","https://openalex.org/W2751746637","https://openalex.org/W2751778071","https://openalex.org/W2752194699","https://openalex.org/W2752332392","https://openalex.org/W2753713840","https://openalex.org/W2804604520","https://openalex.org/W2806816458","https://openalex.org/W2883424428","https://openalex.org/W2888723145","https://openalex.org/W2891130433","https://openalex.org/W2892051885","https://openalex.org/W2892093980","https://openalex.org/W2896774728","https://openalex.org/W2897083899","https://openalex.org/W2905763413","https://openalex.org/W2910705748","https://openalex.org/W2921712231","https://openalex.org/W2949858875","https://openalex.org/W2949973186","https://openalex.org/W2956281901","https://openalex.org/W2963123635","https://openalex.org/W2963214037","https://openalex.org/W2964200170","https://openalex.org/W2969224142","https://openalex.org/W2969940513","https://openalex.org/W2969954596","https://openalex.org/W2972458663","https://openalex.org/W2972615418","https://openalex.org/W2973221207","https://openalex.org/W3008058947","https://openalex.org/W3019489177","https://openalex.org/W3021138682","https://openalex.org/W3021941048","https://openalex.org/W3042781247","https://openalex.org/W3043397348","https://openalex.org/W3048687239","https://openalex.org/W3085905482","https://openalex.org/W3092732263","https://openalex.org/W3092775028","https://openalex.org/W3093958547","https://openalex.org/W3108010663","https://openalex.org/W3110515458","https://openalex.org/W3120379296","https://openalex.org/W3122778363","https://openalex.org/W3148605487","https://openalex.org/W3148955003","https://openalex.org/W3159394994","https://openalex.org/W3173924498","https://openalex.org/W3174542393","https://openalex.org/W3176125472","https://openalex.org/W3197347140","https://openalex.org/W3203598645","https://openalex.org/W3204925778","https://openalex.org/W3216568117","https://openalex.org/W3216607306","https://openalex.org/W3217243150","https://openalex.org/W4210513286","https://openalex.org/W4221165557","https://openalex.org/W4281956361","https://openalex.org/W4288058769","https://openalex.org/W4297252763","https://openalex.org/W4297459708","https://openalex.org/W4298326701","https://openalex.org/W4302275297","https://openalex.org/W4313250813","https://openalex.org/W4366548422","https://openalex.org/W6602366985","https://openalex.org/W6637373629","https://openalex.org/W6730169232","https://openalex.org/W6748854608","https://openalex.org/W6782973736","https://openalex.org/W6802813392","https://openalex.org/W6963358769"],"related_works":["https://openalex.org/W2045049461","https://openalex.org/W4381094582","https://openalex.org/W1978893398","https://openalex.org/W2118717649","https://openalex.org/W1977906818","https://openalex.org/W2201908702","https://openalex.org/W2369625323","https://openalex.org/W2364579609","https://openalex.org/W410723623","https://openalex.org/W2413243053"],"abstract_inverted_index":{"Visualization":[0],"for":[1,90],"machine":[2,22],"learning":[3,23],"(VIS4ML)":[4],"research":[5,68,87,154],"aims":[6],"to":[7,14,36,43,63,115,133,147,160],"help":[8],"experts":[9,128],"apply":[10],"their":[11],"prior":[12],"knowledge":[13,35],"develop,":[15],"understand,":[16],"and":[17,48,51,70,88,129,137,153,156],"improve":[18],"the":[19,31,53,65,82,107,149,167],"performance":[20],"of":[21,33,55,67,85,126,170],"models.":[24],"In":[25],"conceiving":[26],"VIS4ML":[27,61,86,102,171],"systems,":[28],"researchers":[29],"characterize":[30],"nature":[32,169],"human":[34],"support":[37],"human-in-the-loop":[38,74],"tasks,":[39],"design":[40],"interactive":[41],"visualizations":[42],"make":[44],"ML":[45,127],"components":[46],"interpretable":[47],"elicit":[49],"knowledge,":[50],"evaluate":[52],"effectiveness":[54],"human-model":[56],"interchange.":[57],"We":[58,95,144],"survey":[59],"recent":[60],"papers":[62,99],"assess":[64],"generalizability":[66],"contributions":[69],"claims":[71,155],"in":[72,93],"enabling":[73],"ML.":[75],"Our":[76],"results":[77],"show":[78],"potential":[79],"gaps":[80],"between":[81,151],"current":[83],"scope":[84],"aspirations":[89,152],"its":[91],"use":[92],"practice.":[94],"find":[96],"that":[97,101,141,164],"while":[98],"motivate":[100],"systems":[103],"are":[104,112,118],"applicable":[105],"beyond":[106],"specific":[108],"conditions":[109],"studied,":[110],"conclusions":[111],"often":[113],"overfitted":[114],"non-representative":[116],"scenarios,":[117],"based":[119],"on":[120,139],"interactions":[121],"with":[122],"a":[123],"small":[124],"set":[125],"well-understood":[130],"datasets,":[131],"fail":[132],"acknowledge":[134,166],"crucial":[135],"dependencies,":[136],"hinge":[138],"decisions":[140],"lack":[142],"justification.":[143],"discuss":[145],"approaches":[146],"close":[148],"gap":[150],"suggest":[157],"documentation":[158],"practices":[159],"report":[161],"generality":[162],"constraints":[163],"better":[165],"exploratory":[168],"research.":[172]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":4}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}