{"id":"https://openalex.org/W2979893369","doi":"https://doi.org/10.1145/3359152","title":"The Principles and Limits of Algorithm-in-the-Loop Decision Making","display_name":"The Principles and Limits of Algorithm-in-the-Loop Decision Making","publication_year":2019,"publication_date":"2019-11-07","ids":{"openalex":"https://openalex.org/W2979893369","doi":"https://doi.org/10.1145/3359152","mag":"2979893369"},"language":"en","primary_location":{"id":"doi:10.1145/3359152","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3359152","pdf_url":null,"source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100610716","display_name":"Ben Green","orcid":"https://orcid.org/0000-0001-7810-908X"},"institutions":[{"id":"https://openalex.org/I2801851002","display_name":"Harvard University Press","ror":"https://ror.org/006v7bf86","country_code":"US","type":"other","lineage":["https://openalex.org/I136199984","https://openalex.org/I2801851002"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ben Green","raw_affiliation_strings":["Harvard University, Cambridge, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Harvard University, Cambridge, MA, USA","institution_ids":["https://openalex.org/I2801851002"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100738419","display_name":"Yiling Chen","orcid":"https://orcid.org/0000-0002-0166-594X"},"institutions":[{"id":"https://openalex.org/I2801851002","display_name":"Harvard University Press","ror":"https://ror.org/006v7bf86","country_code":"US","type":"other","lineage":["https://openalex.org/I136199984","https://openalex.org/I2801851002"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yiling Chen","raw_affiliation_strings":["Harvard University, Cambridge, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Harvard University, Cambridge, MA, USA","institution_ids":["https://openalex.org/I2801851002"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I2801851002"],"apc_list":null,"apc_paid":null,"fwci":37.6988,"has_fulltext":false,"cited_by_count":364,"citation_normalized_percentile":{"value":0.99834051,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":"3","issue":"CSCW","first_page":"1","last_page":"24"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9962999820709229,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9577999711036682,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/normative","display_name":"Normative","score":0.737648069858551},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5968990325927734},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5419363379478455},{"id":"https://openalex.org/keywords/affect","display_name":"Affect (linguistics)","score":0.5178502202033997},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5154510140419006},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5080420970916748},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.47945061326026917},{"id":"https://openalex.org/keywords/structuring","display_name":"Structuring","score":0.45721718668937683},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.41365641355514526},{"id":"https://openalex.org/keywords/normative-model-of-decision-making","display_name":"Normative model of decision-making","score":0.41223087906837463},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.38109585642814636},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.3266984224319458},{"id":"https://openalex.org/keywords/epistemology","display_name":"Epistemology","score":0.13087797164916992},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.13058680295944214},{"id":"https://openalex.org/keywords/law","display_name":"Law","score":0.11798095703125}],"concepts":[{"id":"https://openalex.org/C44725695","wikidata":"https://www.wikidata.org/wiki/Q288156","display_name":"Normative","level":2,"score":0.737648069858551},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5968990325927734},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5419363379478455},{"id":"https://openalex.org/C2776035688","wikidata":"https://www.wikidata.org/wiki/Q1606558","display_name":"Affect (linguistics)","level":2,"score":0.5178502202033997},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5154510140419006},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5080420970916748},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.47945061326026917},{"id":"https://openalex.org/C2775945657","wikidata":"https://www.wikidata.org/wiki/Q381442","display_name":"Structuring","level":2,"score":0.45721718668937683},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.41365641355514526},{"id":"https://openalex.org/C137549413","wikidata":"https://www.wikidata.org/wiki/Q7053127","display_name":"Normative model of decision-making","level":3,"score":0.41223087906837463},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.38109585642814636},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3266984224319458},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.13087797164916992},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.13058680295944214},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.11798095703125},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C46312422","wikidata":"https://www.wikidata.org/wiki/Q11024","display_name":"Communication","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3359152","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3359152","pdf_url":null,"source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions","score":0.800000011920929}],"awards":[{"id":"https://openalex.org/G4973792023","display_name":"Graduate Research Fellowship Program (GRFP)","funder_award_id":"1745303","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":69,"referenced_works":["https://openalex.org/W12028179","https://openalex.org/W303716618","https://openalex.org/W359283280","https://openalex.org/W1526225581","https://openalex.org/W1553406149","https://openalex.org/W1590711465","https://openalex.org/W1678356000","https://openalex.org/W1989630473","https://openalex.org/W2003238113","https://openalex.org/W2004884942","https://openalex.org/W2011749266","https://openalex.org/W2025486241","https://openalex.org/W2025720061","https://openalex.org/W2059216172","https://openalex.org/W2109021302","https://openalex.org/W2109141422","https://openalex.org/W2110171129","https://openalex.org/W2121042192","https://openalex.org/W2138795819","https://openalex.org/W2157020070","https://openalex.org/W2162409443","https://openalex.org/W2249383713","https://openalex.org/W2282821441","https://openalex.org/W2462893817","https://openalex.org/W2499811908","https://openalex.org/W2539402368","https://openalex.org/W2551317447","https://openalex.org/W2606411279","https://openalex.org/W2618631885","https://openalex.org/W2737089608","https://openalex.org/W2744883502","https://openalex.org/W2751772126","https://openalex.org/W2786004891","https://openalex.org/W2790165607","https://openalex.org/W2792183666","https://openalex.org/W2795850430","https://openalex.org/W2800068874","https://openalex.org/W2888833318","https://openalex.org/W2890416412","https://openalex.org/W2890440562","https://openalex.org/W2891340972","https://openalex.org/W2892657864","https://openalex.org/W2901481055","https://openalex.org/W2901895173","https://openalex.org/W2902302393","https://openalex.org/W2913854599","https://openalex.org/W2922720375","https://openalex.org/W2929545329","https://openalex.org/W2935502729","https://openalex.org/W2942399136","https://openalex.org/W2951111372","https://openalex.org/W2954916332","https://openalex.org/W3099742594","https://openalex.org/W3101038122","https://openalex.org/W3104119469","https://openalex.org/W3122002830","https://openalex.org/W3124480946","https://openalex.org/W3125132731","https://openalex.org/W3125200493","https://openalex.org/W3125633690","https://openalex.org/W3153147958","https://openalex.org/W3217005382","https://openalex.org/W4243342770","https://openalex.org/W4285719527","https://openalex.org/W4296978576","https://openalex.org/W4300071243","https://openalex.org/W4403865180","https://openalex.org/W6688548183","https://openalex.org/W6742636707"],"related_works":["https://openalex.org/W1530384998","https://openalex.org/W2034078532","https://openalex.org/W2470690987","https://openalex.org/W2141923480","https://openalex.org/W2474607346","https://openalex.org/W2969279346","https://openalex.org/W2022375696","https://openalex.org/W2625312343","https://openalex.org/W2021972201","https://openalex.org/W3080883619"],"abstract_inverted_index":{"The":[0],"rise":[1],"of":[2,107,119,168,227,272],"machine":[3,28,83],"learning":[4,29,84],"has":[5,36],"fundamentally":[6],"altered":[7],"decision":[8,143,274],"making:":[9],"rather":[10],"than":[11],"being":[12],"made":[13,22],"solely":[14],"by":[15],"people,":[16],"many":[17],"important":[18],"decisions":[19,120],"are":[20],"now":[21],"through":[23,146],"an":[24],"\"algorithm-in-the-loop''":[25],"process":[26],"where":[27],"models":[30,44,85,95,109,280],"inform":[31],"people.":[32],"Yet":[33],"insufficient":[34],"research":[35,76,127],"considered":[37],"how":[38,56,69,104],"the":[39,115,166,225,231,242,247,259,265,279,284],"interactions":[40,113,257],"between":[41],"people":[42,57,70,79,158,290],"and":[43,86,110,117,140,181,184,192,201,251,291],"actually":[45],"influence":[46,198],"human":[47,174],"decisions.":[48],"Society":[49],"lacks":[50],"both":[51],"clear":[52],"normative":[53],"principles":[54,135,161],"regarding":[55],"should":[58],"collaborate":[59,72],"with":[60,73,165,258],"algorithms":[61,292],"as":[62,64,136],"well":[63],"robust":[65],"empirical":[66],"evidence":[67],"about":[68],"do":[71],"algorithms.":[74],"Given":[75],"suggesting":[77],"that":[78],"struggle":[80],"to":[81,87,96,102,125,138,222,268,282],"interpret":[82],"incorporate":[88],"them":[89],"into":[90],"their":[91,228,239,256],"decisions---sometimes":[92],"leading":[93],"these":[94,160,195],"produce":[97],"unexpected":[98],"outcomes---it":[99],"is":[100],"essential":[101,137],"consider":[103],"different":[105],"ways":[106],"presenting":[108],"structuring":[111],"human-algorithm":[112],"affect":[114],"quality":[116],"type":[118],"made.":[121],"This":[122],"paper":[123],"contributes":[124],"such":[126],"in":[128,176,202,255,288],"two":[129,177],"ways.":[130],"First,":[131],"we":[132,155],"posited":[133],"three":[134],"ethical":[139],"responsible":[141],"algorithm-in-the-loop":[142],"making.":[144],"Second,":[145],"a":[147,169],"controlled":[148],"experimental":[149],"study":[150,217],"on":[151,241,246],"Amazon":[152],"Mechanical":[153],"Turk,":[154],"evaluated":[156],"whether":[157],"satisfy":[159],"when":[162],"making":[163,275],"predictions":[164,175],"aid":[167],"risk":[170,189,232,243,248,260],"assessment.":[171,261],"We":[172],"studied":[173],"contexts":[178,287],"(pretrial":[179],"release":[180],"financial":[182],"lending)":[183],"under":[185],"several":[186],"conditions":[187,196],"for":[188],"assessment":[190,244],"presentation":[191],"structure.":[193],"Although":[194],"did":[197,236],"participant":[199],"behaviors":[200],"some":[203],"cases":[204],"improved":[205],"performance,":[206,250],"only":[207],"one":[208],"desideratum":[209],"was":[210],"consistently":[211],"satisfied.":[212],"Under":[213],"all":[214],"conditions,":[215],"our":[216,270],"participants":[218],"1)":[219],"were":[220],"unable":[221],"effectively":[223],"evaluate":[224],"accuracy":[226],"own":[229],"or":[230],"assessment's":[233,249],"predictions,":[234],"2)":[235],"not":[237],"calibrate":[238],"reliance":[240],"based":[245],"3)":[252],"exhibited":[253],"bias":[254],"These":[262],"results":[263],"highlight":[264],"urgent":[266],"need":[267],"expand":[269],"analyses":[271],"algorithmic":[273],"aids":[276],"beyond":[277],"evaluating":[278],"themselves":[281],"investigating":[283],"full":[285],"sociotechnical":[286],"which":[289],"interact.":[293]},"counts_by_year":[{"year":2026,"cited_by_count":66},{"year":2025,"cited_by_count":60},{"year":2024,"cited_by_count":60},{"year":2023,"cited_by_count":55},{"year":2022,"cited_by_count":57},{"year":2021,"cited_by_count":38},{"year":2020,"cited_by_count":23},{"year":2019,"cited_by_count":5}],"updated_date":"2026-08-28T12:50:07.497085","created_date":"2025-10-10T00:00:00"}