{"id":"https://openalex.org/W4366547794","doi":"https://doi.org/10.1145/3544548.3581099","title":"Understanding and Supporting Debugging Workflows in Multiverse Analysis","display_name":"Understanding and Supporting Debugging Workflows in Multiverse Analysis","publication_year":2023,"publication_date":"2023-04-19","ids":{"openalex":"https://openalex.org/W4366547794","doi":"https://doi.org/10.1145/3544548.3581099"},"language":"en","primary_location":{"id":"doi:10.1145/3544548.3581099","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3544548.3581099","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3544548.3581099","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3544548.3581099","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5001307316","display_name":"Ken Gu","orcid":"https://orcid.org/0000-0002-4343-1578"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ken Gu","raw_affiliation_strings":["Allen School of Computer Science & Engineering, University of Washington, United States"],"raw_orcid":"https://orcid.org/0000-0002-4343-1578","affiliations":[{"raw_affiliation_string":"Allen School of Computer Science & Engineering, University of Washington, United States","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031372138","display_name":"Eunice Jun","orcid":"https://orcid.org/0000-0002-4050-4284"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Eunice Jun","raw_affiliation_strings":["Allen School of Computer Science & Engineering, University of Washington, United States"],"raw_orcid":"https://orcid.org/0000-0002-4050-4284","affiliations":[{"raw_affiliation_string":"Allen School of Computer Science & Engineering, University of Washington, United States","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5033639139","display_name":"Tim Althoff","orcid":"https://orcid.org/0000-0003-4793-2289"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tim Althoff","raw_affiliation_strings":["Allen School of Computer Science & Engineering, University of Washington, United States"],"raw_orcid":"https://orcid.org/0000-0003-4793-2289","affiliations":[{"raw_affiliation_string":"Allen School of Computer Science & Engineering, University of Washington, United States","institution_ids":["https://openalex.org/I201448701"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I201448701"],"apc_list":null,"apc_paid":null,"fwci":4.7001,"has_fulltext":true,"cited_by_count":11,"citation_normalized_percentile":{"value":0.95108916,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"19"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10799","display_name":"Data Visualization and Analytics","score":0.9955999851226807,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9943000078201294,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/debugging","display_name":"Debugging","score":0.9140627384185791},{"id":"https://openalex.org/keywords/debugger","display_name":"Debugger","score":0.8590869903564453},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7757831811904907},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.7308124899864197},{"id":"https://openalex.org/keywords/metadata","display_name":"Metadata","score":0.5591633319854736},{"id":"https://openalex.org/keywords/interface","display_name":"Interface (matter)","score":0.4689198136329651},{"id":"https://openalex.org/keywords/transparency","display_name":"Transparency (behavior)","score":0.4269666373729706},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.3752940893173218},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.3749956488609314},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.34776031970977783},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.17352426052093506},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.13035303354263306},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.10658881068229675},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.08258557319641113}],"concepts":[{"id":"https://openalex.org/C168065819","wikidata":"https://www.wikidata.org/wiki/Q845566","display_name":"Debugging","level":2,"score":0.9140627384185791},{"id":"https://openalex.org/C2778485113","wikidata":"https://www.wikidata.org/wiki/Q193231","display_name":"Debugger","level":3,"score":0.8590869903564453},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7757831811904907},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.7308124899864197},{"id":"https://openalex.org/C93518851","wikidata":"https://www.wikidata.org/wiki/Q180160","display_name":"Metadata","level":2,"score":0.5591633319854736},{"id":"https://openalex.org/C113843644","wikidata":"https://www.wikidata.org/wiki/Q901882","display_name":"Interface (matter)","level":4,"score":0.4689198136329651},{"id":"https://openalex.org/C2780233690","wikidata":"https://www.wikidata.org/wiki/Q535347","display_name":"Transparency (behavior)","level":2,"score":0.4269666373729706},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3752940893173218},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3749956488609314},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.34776031970977783},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.17352426052093506},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.13035303354263306},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.10658881068229675},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.08258557319641113},{"id":"https://openalex.org/C157915830","wikidata":"https://www.wikidata.org/wiki/Q2928001","display_name":"Bubble","level":2,"score":0.0},{"id":"https://openalex.org/C129307140","wikidata":"https://www.wikidata.org/wiki/Q6795880","display_name":"Maximum bubble pressure method","level":3,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3544548.3581099","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3544548.3581099","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3544548.3581099","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3544548.3581099","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3544548.3581099","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3544548.3581099","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/5","display_name":"Gender equality","score":0.4699999988079071}],"awards":[{"id":"https://openalex.org/G1193584625","display_name":null,"funder_award_id":"R01MH125179","funder_id":"https://openalex.org/F4320332161","funder_display_name":"National Institutes of Health"},{"id":"https://openalex.org/G1465050376","display_name":"CAREER: Realizing the Potential of Behavioral Data Science for Population Health","funder_award_id":"2142794","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G3398026002","display_name":"EAGER: Partnering with Grab for Cross-ASEAN Regional Urban Insights","funder_award_id":"2025022","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G3506071450","display_name":"University of Washington Foundation - to develop new machine learning approaches to understand human health","funder_award_id":"INV-004841","funder_id":"https://openalex.org/F4320306137","funder_display_name":"Bill and Melinda Gates Foundation"},{"id":"https://openalex.org/G4007484515","display_name":null,"funder_award_id":"CNS-2025022","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5446961646","display_name":null,"funder_award_id":"IIS-1901386","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6671297155","display_name":null,"funder_award_id":"CAREER","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7520803678","display_name":null,"funder_award_id":"N00014-21-1-2154","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"},{"id":"https://openalex.org/G8529959211","display_name":"III: Large: Collaborative Research: Analysis Engineering for Robust End-to-End Data Science","funder_award_id":"1901386","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8876996369","display_name":null,"funder_award_id":"N00014","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320306137","display_name":"Bill and Melinda Gates Foundation","ror":"https://ror.org/0456r8d26"},{"id":"https://openalex.org/F4320332161","display_name":"National Institutes of Health","ror":"https://ror.org/01cwqze88"},{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4366547794.pdf","grobid_xml":"https://content.openalex.org/works/W4366547794.grobid-xml"},"referenced_works_count":38,"referenced_works":["https://openalex.org/W1596127723","https://openalex.org/W1897139626","https://openalex.org/W1991202814","https://openalex.org/W2065489029","https://openalex.org/W2151996389","https://openalex.org/W2157931132","https://openalex.org/W2174191405","https://openalex.org/W2401610261","https://openalex.org/W2406493898","https://openalex.org/W2526501380","https://openalex.org/W2610517421","https://openalex.org/W2741561716","https://openalex.org/W2787804922","https://openalex.org/W2794673032","https://openalex.org/W2795017501","https://openalex.org/W2807775361","https://openalex.org/W2898377472","https://openalex.org/W2919957599","https://openalex.org/W2927052147","https://openalex.org/W2934604240","https://openalex.org/W2936436943","https://openalex.org/W2939684257","https://openalex.org/W2941088780","https://openalex.org/W2962851501","https://openalex.org/W2982550137","https://openalex.org/W3031170804","https://openalex.org/W3041984699","https://openalex.org/W3044225844","https://openalex.org/W3102186774","https://openalex.org/W3169848230","https://openalex.org/W4210453740","https://openalex.org/W4225085525","https://openalex.org/W4247854836","https://openalex.org/W4288068563","https://openalex.org/W4302578348","https://openalex.org/W4307475647","https://openalex.org/W6946066278","https://openalex.org/W7011718474"],"related_works":["https://openalex.org/W2800641345","https://openalex.org/W2886673879","https://openalex.org/W564953056","https://openalex.org/W2061184006","https://openalex.org/W2381239448","https://openalex.org/W4236910510","https://openalex.org/W2092932442","https://openalex.org/W2543963479","https://openalex.org/W1751798423","https://openalex.org/W2583000271"],"abstract_inverted_index":{"Multiverse":[0,77,97],"analysis\u2014a":[1],"paradigm":[2],"for":[3,125],"statistical":[4],"analysis":[5,12,128],"that":[6],"considers":[7],"all":[8],"combinations":[9],"of":[10,59,106],"reasonable":[11],"choices":[13],"in":[14,34,83,115],"parallel\u2014promises":[15],"to":[16,32,47,53,63,102],"improve":[17],"transparency":[18],"and":[19,56,86,109],"reproducibility.":[20],"Although":[21],"recent":[22],"tools":[23],"help":[24],"analysts":[25],"specify":[26],"multiverse":[27,85,127],"analyses,":[28],"they":[29],"remain":[30],"difficult":[31],"use":[33,96],"practice.":[35],"In":[36,89],"this":[37],"work,":[38],"we":[39,71,95],"identify":[40,110],"debugging":[41,107],"as":[42,99],"a":[43,65,73,90,100,104],"key":[44],"barrier":[45],"due":[46],"the":[48,57,84,117],"latency":[49],"from":[50],"running":[51],"analyses":[52],"detecting":[54],"bugs":[55,82],"scale":[58],"metadata":[60],"processing":[61],"needed":[62],"diagnose":[64,81],"bug.":[66],"To":[67],"address":[68],"these":[69],"challenges,":[70,112],"prototype":[72],"command-line":[74],"interface":[75],"tool,":[76],"Debugger,":[78],"which":[79],"helps":[80],"propagate":[87],"fixes.":[88],"qualitative":[91],"lab":[92],"study":[93],"(n=13),":[94],"Debugger":[98],"probe":[101],"develop":[103],"model":[105],"workflows":[108],"specific":[111],"including":[113],"difficulty":[114],"understanding":[116],"multiverse\u2019s":[118],"composition.":[119],"We":[120],"conclude":[121],"with":[122],"design":[123],"implications":[124],"future":[126],"authoring":[129],"systems.":[130]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":3}],"updated_date":"2026-06-26T08:34:08.712188","created_date":"2025-10-10T00:00:00"}