{"id":"https://openalex.org/W4365597197","doi":"https://doi.org/10.1145/3544548.3580790","title":"Angler: Helping Machine Translation Practitioners Prioritize Model Improvements","display_name":"Angler: Helping Machine Translation Practitioners Prioritize Model Improvements","publication_year":2023,"publication_date":"2023-04-19","ids":{"openalex":"https://openalex.org/W4365597197","doi":"https://doi.org/10.1145/3544548.3580790"},"language":"en","primary_location":{"id":"doi:10.1145/3544548.3580790","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3544548.3580790","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3544548.3580790","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3544548.3580790","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5073527485","display_name":"Samantha Robertson","orcid":"https://orcid.org/0000-0002-0386-4555"},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Samantha Robertson","raw_affiliation_strings":["Electrical Engineering and Computer Sciences, University of California, Berkeley, United States"],"raw_orcid":"https://orcid.org/0000-0002-0386-4555","affiliations":[{"raw_affiliation_string":"Electrical Engineering and Computer Sciences, University of California, Berkeley, United States","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100638703","display_name":"Zijie J. Wang","orcid":"https://orcid.org/0000-0003-4360-1423"},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]},{"id":"https://openalex.org/I2800444561","display_name":"Atlanta Technical College","ror":"https://ror.org/01s3vfp47","country_code":"US","type":"education","lineage":["https://openalex.org/I2800444561"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zijie J. Wang","raw_affiliation_strings":["College of Computing, Georgia Tech, United States"],"raw_orcid":"https://orcid.org/0000-0003-4360-1423","affiliations":[{"raw_affiliation_string":"College of Computing, Georgia Tech, United States","institution_ids":["https://openalex.org/I130701444","https://openalex.org/I2800444561"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006087928","display_name":"Dominik Moritz","orcid":"https://orcid.org/0000-0002-3110-1053"},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dominik Moritz","raw_affiliation_strings":["Apple, United States"],"raw_orcid":"https://orcid.org/0000-0002-3110-1053","affiliations":[{"raw_affiliation_string":"Apple, United States","institution_ids":["https://openalex.org/I4210153776"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068961297","display_name":"Mary Beth Kery","orcid":"https://orcid.org/0000-0002-1771-0565"},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mary Beth Kery","raw_affiliation_strings":["Apple Inc., United States"],"raw_orcid":"https://orcid.org/0000-0002-1771-0565","affiliations":[{"raw_affiliation_string":"Apple Inc., United States","institution_ids":["https://openalex.org/I4210153776"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5034842879","display_name":"Fred Hohman","orcid":null},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Fred Hohman","raw_affiliation_strings":["Apple, United States"],"raw_orcid":"https://orcid.org/0000-0002-4164-844X","affiliations":[{"raw_affiliation_string":"Apple, United States","institution_ids":["https://openalex.org/I4210153776"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.4276,"has_fulltext":true,"cited_by_count":20,"citation_normalized_percentile":{"value":0.9260198,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"20"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10799","display_name":"Data Visualization and Analytics","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10799","display_name":"Data Visualization and Analytics","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9866999983787537,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7824219465255737},{"id":"https://openalex.org/keywords/scope","display_name":"Scope (computer science)","score":0.6864453554153442},{"id":"https://openalex.org/keywords/debugging","display_name":"Debugging","score":0.6341369152069092},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.6153059005737305},{"id":"https://openalex.org/keywords/machine-translation","display_name":"Machine translation","score":0.580410361289978},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5502569079399109},{"id":"https://openalex.org/keywords/prioritization","display_name":"Prioritization","score":0.5413503646850586},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.5053279995918274},{"id":"https://openalex.org/keywords/space","display_name":"Space (punctuation)","score":0.4936438798904419},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4699815809726715},{"id":"https://openalex.org/keywords/analytics","display_name":"Analytics","score":0.4362866282463074},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4265308380126953},{"id":"https://openalex.org/keywords/reading","display_name":"Reading (process)","score":0.4257868528366089},{"id":"https://openalex.org/keywords/workbench","display_name":"Workbench","score":0.41646242141723633},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.33094680309295654},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.19351831078529358},{"id":"https://openalex.org/keywords/process-management","display_name":"Process management","score":0.16500544548034668},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.08998170495033264},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.08765709400177002}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7824219465255737},{"id":"https://openalex.org/C2778012447","wikidata":"https://www.wikidata.org/wiki/Q1034415","display_name":"Scope (computer science)","level":2,"score":0.6864453554153442},{"id":"https://openalex.org/C168065819","wikidata":"https://www.wikidata.org/wiki/Q845566","display_name":"Debugging","level":2,"score":0.6341369152069092},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6153059005737305},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.580410361289978},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5502569079399109},{"id":"https://openalex.org/C2777615720","wikidata":"https://www.wikidata.org/wiki/Q11888847","display_name":"Prioritization","level":2,"score":0.5413503646850586},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.5053279995918274},{"id":"https://openalex.org/C2778572836","wikidata":"https://www.wikidata.org/wiki/Q380933","display_name":"Space (punctuation)","level":2,"score":0.4936438798904419},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4699815809726715},{"id":"https://openalex.org/C79158427","wikidata":"https://www.wikidata.org/wiki/Q485396","display_name":"Analytics","level":2,"score":0.4362866282463074},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4265308380126953},{"id":"https://openalex.org/C554936623","wikidata":"https://www.wikidata.org/wiki/Q199657","display_name":"Reading (process)","level":2,"score":0.4257868528366089},{"id":"https://openalex.org/C2779145975","wikidata":"https://www.wikidata.org/wiki/Q347690","display_name":"Workbench","level":3,"score":0.41646242141723633},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.33094680309295654},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.19351831078529358},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.16500544548034668},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.08998170495033264},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.08765709400177002},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3544548.3580790","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3544548.3580790","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3544548.3580790","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2304.05967","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2304.05967","pdf_url":"https://arxiv.org/pdf/2304.05967","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3544548.3580790","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3544548.3580790","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3544548.3580790","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.8500000238418579,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4365597197.pdf","grobid_xml":"https://content.openalex.org/works/W4365597197.grobid-xml"},"referenced_works_count":138,"referenced_works":["https://openalex.org/W69214955","https://openalex.org/W245566120","https://openalex.org/W1489525520","https://openalex.org/W1552182777","https://openalex.org/W1593271688","https://openalex.org/W1995923922","https://openalex.org/W2029135525","https://openalex.org/W2034280257","https://openalex.org/W2063011253","https://openalex.org/W2098865355","https://openalex.org/W2101234009","https://openalex.org/W2108082276","https://openalex.org/W2108816886","https://openalex.org/W2123301721","https://openalex.org/W2129905273","https://openalex.org/W2135415614","https://openalex.org/W2142112143","https://openalex.org/W2147192413","https://openalex.org/W2165558283","https://openalex.org/W2187089797","https://openalex.org/W2189162242","https://openalex.org/W2242404765","https://openalex.org/W2250342921","https://openalex.org/W2252029987","https://openalex.org/W2282821441","https://openalex.org/W2294798173","https://openalex.org/W2330219538","https://openalex.org/W2463396630","https://openalex.org/W2514499445","https://openalex.org/W2550821151","https://openalex.org/W2564739834","https://openalex.org/W2606134370","https://openalex.org/W2614322402","https://openalex.org/W2622101571","https://openalex.org/W2751722336","https://openalex.org/W2754210855","https://openalex.org/W2756888147","https://openalex.org/W2757188127","https://openalex.org/W2760327630","https://openalex.org/W2771976988","https://openalex.org/W2786672974","https://openalex.org/W2788481061","https://openalex.org/W2806532810","https://openalex.org/W2807434717","https://openalex.org/W2898962441","https://openalex.org/W2900211842","https://openalex.org/W2915231456","https://openalex.org/W2918341242","https://openalex.org/W2922234936","https://openalex.org/W2922349260","https://openalex.org/W2946595616","https://openalex.org/W2948897128","https://openalex.org/W2948917818","https://openalex.org/W2949858875","https://openalex.org/W2950018712","https://openalex.org/W2952328691","https://openalex.org/W2953937638","https://openalex.org/W2956281901","https://openalex.org/W2962862931","https://openalex.org/W2963123635","https://openalex.org/W2963198001","https://openalex.org/W2963214037","https://openalex.org/W2963323070","https://openalex.org/W2963394326","https://openalex.org/W2963457723","https://openalex.org/W2963526187","https://openalex.org/W2963532001","https://openalex.org/W2964130895","https://openalex.org/W2965446359","https://openalex.org/W2970442950","https://openalex.org/W2970641574","https://openalex.org/W2971141904","https://openalex.org/W2981869278","https://openalex.org/W2985337504","https://openalex.org/W3004542466","https://openalex.org/W3006437051","https://openalex.org/W3012624518","https://openalex.org/W3030081171","https://openalex.org/W3031531573","https://openalex.org/W3035408261","https://openalex.org/W3035507081","https://openalex.org/W3038035611","https://openalex.org/W3082928416","https://openalex.org/W3088581407","https://openalex.org/W3092425241","https://openalex.org/W3093271375","https://openalex.org/W3099444752","https://openalex.org/W3103029570","https://openalex.org/W3105773826","https://openalex.org/W3105928338","https://openalex.org/W3112486745","https://openalex.org/W3119000810","https://openalex.org/W3125603643","https://openalex.org/W3126623050","https://openalex.org/W3135453287","https://openalex.org/W3153978352","https://openalex.org/W3159892921","https://openalex.org/W3160887439","https://openalex.org/W3166396011","https://openalex.org/W3171654528","https://openalex.org/W3174057701","https://openalex.org/W3175487198","https://openalex.org/W3182543254","https://openalex.org/W3184545100","https://openalex.org/W3184924454","https://openalex.org/W3185106120","https://openalex.org/W3204393347","https://openalex.org/W3205290952","https://openalex.org/W3206420877","https://openalex.org/W3212173134","https://openalex.org/W3213728774","https://openalex.org/W4200296112","https://openalex.org/W4200475213","https://openalex.org/W4206256378","https://openalex.org/W4214895166","https://openalex.org/W4221142221","https://openalex.org/W4223555496","https://openalex.org/W4224992683","https://openalex.org/W4226148503","https://openalex.org/W4226182712","https://openalex.org/W4243863038","https://openalex.org/W4245267204","https://openalex.org/W4281846725","https://openalex.org/W4283269716","https://openalex.org/W4285113702","https://openalex.org/W4286891940","https://openalex.org/W4287802469","https://openalex.org/W4288359139","https://openalex.org/W4288359825","https://openalex.org/W4297816198","https://openalex.org/W4298326701","https://openalex.org/W4308784155","https://openalex.org/W4312620238","https://openalex.org/W4313068521","https://openalex.org/W4313119036","https://openalex.org/W4318225411","https://openalex.org/W4372229493","https://openalex.org/W4391156274"],"related_works":["https://openalex.org/W2388509490","https://openalex.org/W2109648960","https://openalex.org/W2050384624","https://openalex.org/W2166015079","https://openalex.org/W2776888796","https://openalex.org/W2388480422","https://openalex.org/W2624268110","https://openalex.org/W2951120646","https://openalex.org/W2356913294","https://openalex.org/W2903672016"],"abstract_inverted_index":{"Machine":[0],"learning":[1],"(ML)":[2],"models":[3],"can":[4],"fail":[5],"in":[6,9,69],"unexpected":[7],"ways":[8],"the":[10,108],"real":[11],"world,":[12],"but":[13],"not":[14],"all":[15],"model":[16,32,89,118],"failures":[17],"are":[18,27],"equal.":[19],"With":[20],"finite":[21],"time":[22],"and":[23,34,60,77,113,131,141],"resources,":[24],"ML":[25,41],"practitioners":[26,42,48,87],"forced":[28],"to":[29,54,85,103],"prioritize":[30,88],"their":[31],"debugging":[33],"improvement":[35],"efforts.":[36],"Through":[37],"interviews":[38],"with":[39,73,95],"13":[40],"at":[43],"Apple,":[44],"we":[45,100],"found":[46],"that":[47,125],"construct":[49],"small":[50],"targeted":[51],"test":[52],"sets":[53],"estimate":[55],"an":[56,80],"error\u2019s":[57],"nature,":[58],"scope,":[59],"impact":[61],"on":[62,66],"users.":[63],"We":[64],"built":[65],"this":[67],"insight":[68],"a":[70,92],"case":[71],"study":[72,94,123],"machine":[74,97],"translation":[75,98],"models,":[76],"developed":[78],"Angler,":[79],"interactive":[81],"visual":[82],"analytics":[83],"tool":[84],"help":[86],"improvements.":[90],"In":[91],"user":[93],"7":[96],"experts,":[99],"used":[101],"Angler":[102],"understand":[104],"prioritization":[105,135],"practices":[106],"when":[107],"input":[109],"space":[110],"is":[111,120],"infinite,":[112],"obtaining":[114],"reliable":[115],"signals":[116],"of":[117],"quality":[119],"expensive.":[121],"Our":[122],"revealed":[124],"participants":[126],"could":[127],"form":[128],"more":[129],"interesting":[130],"user-focused":[132],"hypotheses":[133],"for":[134],"by":[136,145],"analyzing":[137],"quantitative":[138],"summary":[139],"statistics":[140],"qualitatively":[142],"assessing":[143],"data":[144],"reading":[146],"sentences.":[147]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":3}],"updated_date":"2026-07-30T17:31:21.811387","created_date":"2023-04-15T00:00:00"}