{"id":"https://openalex.org/W2963678934","doi":"https://doi.org/10.1145/3209889.3209897","title":"Accelerating Human-in-the-loop Machine Learning","display_name":"Accelerating Human-in-the-loop Machine Learning","publication_year":2018,"publication_date":"2018-05-14","ids":{"openalex":"https://openalex.org/W2963678934","doi":"https://doi.org/10.1145/3209889.3209897","mag":"2963678934"},"language":"en","primary_location":{"id":"doi:10.1145/3209889.3209897","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3209889.3209897","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Second Workshop on Data Management for End-To-End Machine Learning","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://www.ideals.illinois.edu/items/112866/bitstreams/369886/data.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5028796772","display_name":"Doris Xin","orcid":null},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Doris Xin","raw_affiliation_strings":["University of Illinois, Urbana-Champaign (UIUC)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois, Urbana-Champaign (UIUC)","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036444718","display_name":"Litian Ma","orcid":null},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Litian Ma","raw_affiliation_strings":["University of Illinois, Urbana-Champaign (UIUC)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois, Urbana-Champaign (UIUC)","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100456650","display_name":"Jialin Liu","orcid":"https://orcid.org/0000-0003-4347-6458"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jialin Liu","raw_affiliation_strings":["University of Illinois, Urbana-Champaign (UIUC)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois, Urbana-Champaign (UIUC)","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072195443","display_name":"Stephen Macke","orcid":null},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Stephen Macke","raw_affiliation_strings":["University of Illinois, Urbana-Champaign (UIUC)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois, Urbana-Champaign (UIUC)","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102970330","display_name":"Shuchen Song","orcid":"https://orcid.org/0000-0003-0875-5160"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shuchen Song","raw_affiliation_strings":["University of Illinois, Urbana-Champaign (UIUC)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois, Urbana-Champaign (UIUC)","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5013608601","display_name":"Aditya Parameswaran","orcid":"https://orcid.org/0000-0002-4538-4752"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aditya Parameswaran","raw_affiliation_strings":["University of Illinois, Urbana-Champaign (UIUC)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois, Urbana-Champaign (UIUC)","institution_ids":["https://openalex.org/I157725225"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I157725225"],"apc_list":null,"apc_paid":null,"fwci":6.7157,"has_fulltext":true,"cited_by_count":120,"citation_normalized_percentile":{"value":0.97408999,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"4"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9965000152587891,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.9926000237464905,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.832452654838562},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.8214703798294067},{"id":"https://openalex.org/keywords/debugging","display_name":"Debugging","score":0.7718105912208557},{"id":"https://openalex.org/keywords/iterative-and-incremental-development","display_name":"Iterative and incremental development","score":0.722379207611084},{"id":"https://openalex.org/keywords/human-in-the-loop","display_name":"Human-in-the-loop","score":0.7078002691268921},{"id":"https://openalex.org/keywords/iterative-learning-control","display_name":"Iterative learning control","score":0.6772049069404602},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.6477488279342651},{"id":"https://openalex.org/keywords/automation","display_name":"Automation","score":0.5968407392501831},{"id":"https://openalex.org/keywords/introspection","display_name":"Introspection","score":0.5746409296989441},{"id":"https://openalex.org/keywords/loop","display_name":"Loop (graph theory)","score":0.5672653317451477},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.390080064535141},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.3606652021408081},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3551034927368164},{"id":"https://openalex.org/keywords/control","display_name":"Control (management)","score":0.21226757764816284},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.1332244575023651},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.0798177421092987}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.832452654838562},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.8214703798294067},{"id":"https://openalex.org/C168065819","wikidata":"https://www.wikidata.org/wiki/Q845566","display_name":"Debugging","level":2,"score":0.7718105912208557},{"id":"https://openalex.org/C143587482","wikidata":"https://www.wikidata.org/wiki/Q1543216","display_name":"Iterative and incremental development","level":2,"score":0.722379207611084},{"id":"https://openalex.org/C2780626000","wikidata":"https://www.wikidata.org/wiki/Q5936775","display_name":"Human-in-the-loop","level":2,"score":0.7078002691268921},{"id":"https://openalex.org/C117619785","wikidata":"https://www.wikidata.org/wiki/Q6094414","display_name":"Iterative learning control","level":3,"score":0.6772049069404602},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.6477488279342651},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.5968407392501831},{"id":"https://openalex.org/C129671850","wikidata":"https://www.wikidata.org/wiki/Q210501","display_name":"Introspection","level":2,"score":0.5746409296989441},{"id":"https://openalex.org/C184670325","wikidata":"https://www.wikidata.org/wiki/Q512604","display_name":"Loop (graph theory)","level":2,"score":0.5672653317451477},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.390080064535141},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3606652021408081},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3551034927368164},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.21226757764816284},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.1332244575023651},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0798177421092987},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.0},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3209889.3209897","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3209889.3209897","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Second Workshop on Data Management for End-To-End Machine Learning","raw_type":"proceedings-article"},{"id":"pmh:oai:www.ideals.illinois.edu:2142/105721","is_oa":true,"landing_page_url":"http://hdl.handle.net/2142/105721","pdf_url":"https://www.ideals.illinois.edu/items/112866/bitstreams/369886/data.pdf","source":{"id":"https://openalex.org/S4377196349","display_name":"IDEALS (University of Illinois Urbana-Champaign)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I157725225","host_organization_name":"University of Illinois Urbana-Champaign","host_organization_lineage":["https://openalex.org/I157725225"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Thesis"}],"best_oa_location":{"id":"pmh:oai:www.ideals.illinois.edu:2142/105721","is_oa":true,"landing_page_url":"http://hdl.handle.net/2142/105721","pdf_url":"https://www.ideals.illinois.edu/items/112866/bitstreams/369886/data.pdf","source":{"id":"https://openalex.org/S4377196349","display_name":"IDEALS (University of Illinois Urbana-Champaign)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I157725225","host_organization_name":"University of Illinois Urbana-Champaign","host_organization_lineage":["https://openalex.org/I157725225"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Thesis"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2963678934.pdf","grobid_xml":"https://content.openalex.org/works/W2963678934.grobid-xml"},"referenced_works_count":14,"referenced_works":["https://openalex.org/W1874755017","https://openalex.org/W2038412523","https://openalex.org/W2099102906","https://openalex.org/W2102458936","https://openalex.org/W2131975293","https://openalex.org/W2184623761","https://openalex.org/W2432911982","https://openalex.org/W2547386789","https://openalex.org/W2611090425","https://openalex.org/W2611130659","https://openalex.org/W2614986686","https://openalex.org/W2949336811","https://openalex.org/W2998715488","https://openalex.org/W4285719527"],"related_works":["https://openalex.org/W4387404794","https://openalex.org/W2380479582","https://openalex.org/W4387405288","https://openalex.org/W4387404506","https://openalex.org/W1674316682","https://openalex.org/W2626546066","https://openalex.org/W2062347313","https://openalex.org/W2797333641","https://openalex.org/W2340943204","https://openalex.org/W2963678934"],"abstract_inverted_index":{"Development":[0],"of":[1,10,82],"machine":[2],"learning":[3],"(ML)":[4],"workflows":[5,18,88],"is":[6,23],"a":[7,30,48,74],"tedious":[8],"process":[9],"iterative":[11,87],"experimentation:":[12],"developers":[13],"repeatedly":[14],"make":[15],"changes":[16,41],"to":[17,80],"until":[19],"the":[20],"desired":[21],"accuracy":[22],"attained.":[24],"We":[25,65],"describe":[26,67],"our":[27,69],"vision":[28],"for":[29],"\"human-in-the-loop\"":[31],"ML":[32],"system":[33,49,75],"that":[34,76],"accelerates":[35],"this":[36],"process:":[37],"by":[38],"intelligently":[39],"tracking":[40],"and":[42,58,60,63],"intermediate":[43],"results":[44],"over":[45],"time,":[46],"such":[47,73],"can":[50],"enable":[51],"rapid":[52],"iteration,":[53],"quick":[54],"responsive":[55],"feedback,":[56],"introspection":[57],"debugging,":[59],"background":[61],"execution":[62],"automation.":[64],"finally":[66],"Helix,":[68],"preliminary":[70],"attempt":[71],"at":[72],"has":[77],"already":[78],"led":[79],"speedups":[81],"upto":[83],"10x":[84],"on":[85],"typical":[86],"against":[89],"competing":[90],"systems.":[91]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":21},{"year":2024,"cited_by_count":21},{"year":2023,"cited_by_count":16},{"year":2022,"cited_by_count":17},{"year":2021,"cited_by_count":20},{"year":2020,"cited_by_count":9},{"year":2019,"cited_by_count":8},{"year":2018,"cited_by_count":4}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}