{"id":"https://openalex.org/W4407240715","doi":"https://doi.org/10.1145/3716134","title":"Designing LLM Chains by Adapting Techniques from Crowdsourcing Workflows","display_name":"Designing LLM Chains by Adapting Techniques from Crowdsourcing Workflows","publication_year":2025,"publication_date":"2025-02-07","ids":{"openalex":"https://openalex.org/W4407240715","doi":"https://doi.org/10.1145/3716134"},"language":"en","primary_location":{"id":"doi:10.1145/3716134","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3716134","pdf_url":null,"source":{"id":"https://openalex.org/S89276529","display_name":"ACM Transactions on Computer-Human Interaction","issn_l":"1073-0516","issn":["1073-0516","1557-7325"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Computer-Human Interaction","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1145/3716134","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5019815997","display_name":"Madeleine Grunde-McLaughlin","orcid":"https://orcid.org/0000-0001-7290-068X"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Madeleine Grunde-McLaughlin","raw_affiliation_strings":["Paul G. Allen School of Computer Science & Engineering, University of Washington, Seattle, Washington, USA","University of Washington, USA"],"affiliations":[{"raw_affiliation_string":"Paul G. Allen School of Computer Science & Engineering, University of Washington, Seattle, Washington, USA","institution_ids":["https://openalex.org/I201448701"]},{"raw_affiliation_string":"University of Washington, USA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101479345","display_name":"Michelle S. Lam","orcid":"https://orcid.org/0000-0002-3448-5961"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michelle S. Lam","raw_affiliation_strings":["Computer Science, Stanford University, Stanford, California, USA","Stanford University, USA"],"affiliations":[{"raw_affiliation_string":"Computer Science, Stanford University, Stanford, California, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"Stanford University, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032451496","display_name":"Ranjay Krishna","orcid":"https://orcid.org/0000-0001-8784-2531"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]},{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]},{"id":"https://openalex.org/I4210156221","display_name":"Allen Institute for Artificial Intelligence","ror":"https://ror.org/05w520734","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210156221"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ranjay Krishna","raw_affiliation_strings":["Paul G. Allen School of Computer Science & Engineering, University of Washington, Seattle, Washington, USA and PRIOR, Allen Institute for Artificial Intelligence, Seattle, Washington, USA","University of Washington, USA and Allen Institute for AI, USA"],"affiliations":[{"raw_affiliation_string":"Paul G. Allen School of Computer Science & Engineering, University of Washington, Seattle, Washington, USA and PRIOR, Allen Institute for Artificial Intelligence, Seattle, Washington, USA","institution_ids":["https://openalex.org/I4210156221"]},{"raw_affiliation_string":"University of Washington, USA and Allen Institute for AI, USA","institution_ids":["https://openalex.org/I4210140341","https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085011940","display_name":"Daniel S. Weld","orcid":"https://orcid.org/0000-0002-3255-0109"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]},{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]},{"id":"https://openalex.org/I4210156221","display_name":"Allen Institute for Artificial Intelligence","ror":"https://ror.org/05w520734","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210156221"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Daniel S. Weld","raw_affiliation_strings":["Semantic Scholar, Allen Institute for Artificial Intelligence, Seattle, Washington, USA and Paul G. Allen School of Computer Science & Engineering, University of Washington, Seattle, Washington, USA","Allen Institute for AI, USA and University of Washington, USA"],"affiliations":[{"raw_affiliation_string":"Semantic Scholar, Allen Institute for Artificial Intelligence, Seattle, Washington, USA and Paul G. Allen School of Computer Science & Engineering, University of Washington, Seattle, Washington, USA","institution_ids":["https://openalex.org/I4210156221"]},{"raw_affiliation_string":"Allen Institute for AI, USA and University of Washington, USA","institution_ids":["https://openalex.org/I4210140341","https://openalex.org/I201448701"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5090570042","display_name":"Jeffrey Heer","orcid":"https://orcid.org/0000-0002-6175-1655"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jeffrey Heer","raw_affiliation_strings":["Paul G. Allen School of Computer Science & Engineering, University of Washington, Seattle, Washington, USA","University of Washington, USA"],"affiliations":[{"raw_affiliation_string":"Paul G. Allen School of Computer Science & Engineering, University of Washington, Seattle, Washington, USA","institution_ids":["https://openalex.org/I201448701"]},{"raw_affiliation_string":"University of Washington, USA","institution_ids":["https://openalex.org/I201448701"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5019815997"],"corresponding_institution_ids":["https://openalex.org/I201448701"],"apc_list":null,"apc_paid":null,"fwci":15.1515,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.98666319,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":"32","issue":"3","first_page":"1","last_page":"57"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10703","display_name":"Business Process Modeling and Analysis","score":0.9937999844551086,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10703","display_name":"Business Process Modeling and Analysis","score":0.9937999844551086,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.9865999817848206,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.9793000221252441,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9544234275817871},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.8260412216186523},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5288226008415222},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4294132590293884},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.2806808650493622},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.18450534343719482}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9544234275817871},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.8260412216186523},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5288226008415222},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4294132590293884},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.2806808650493622},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.18450534343719482}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3716134","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3716134","pdf_url":null,"source":{"id":"https://openalex.org/S89276529","display_name":"ACM Transactions on Computer-Human Interaction","issn_l":"1073-0516","issn":["1073-0516","1557-7325"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Computer-Human Interaction","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3716134","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3716134","pdf_url":null,"source":{"id":"https://openalex.org/S89276529","display_name":"ACM Transactions on Computer-Human Interaction","issn_l":"1073-0516","issn":["1073-0516","1557-7325"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Computer-Human Interaction","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G6173629864","display_name":null,"funder_award_id":"N00014-21-1-2707","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"}],"funders":[{"id":"https://openalex.org/F4320317052","display_name":"Allen Institute for Artificial Intelligence","ror":"https://ror.org/05w520734"},{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":129,"referenced_works":["https://openalex.org/W1516019649","https://openalex.org/W1517053320","https://openalex.org/W1527457771","https://openalex.org/W1537346517","https://openalex.org/W1607035479","https://openalex.org/W1649645444","https://openalex.org/W1974736281","https://openalex.org/W1979290264","https://openalex.org/W1985816302","https://openalex.org/W1988511622","https://openalex.org/W2005639687","https://openalex.org/W2028953510","https://openalex.org/W2036625304","https://openalex.org/W2049774111","https://openalex.org/W2050491563","https://openalex.org/W2053859636","https://openalex.org/W2059216172","https://openalex.org/W2068645020","https://openalex.org/W2078429476","https://openalex.org/W2099531266","https://openalex.org/W2112021024","https://openalex.org/W2115647291","https://openalex.org/W2118153153","https://openalex.org/W2120396827","https://openalex.org/W2122885251","https://openalex.org/W2124793952","https://openalex.org/W2127008633","https://openalex.org/W2133485007","https://openalex.org/W2148479118","https://openalex.org/W2151785234","https://openalex.org/W2152161678","https://openalex.org/W2163986367","https://openalex.org/W2166145477","https://openalex.org/W2206440038","https://openalex.org/W2249218632","https://openalex.org/W2262972580","https://openalex.org/W2396078445","https://openalex.org/W2404427926","https://openalex.org/W2406037548","https://openalex.org/W2469104253","https://openalex.org/W2588579248","https://openalex.org/W2607311634","https://openalex.org/W2610281177","https://openalex.org/W2766913512","https://openalex.org/W2768343520","https://openalex.org/W2769533150","https://openalex.org/W2783539613","https://openalex.org/W2888482885","https://openalex.org/W2911280963","https://openalex.org/W2912745464","https://openalex.org/W2913593702","https://openalex.org/W2913841425","https://openalex.org/W2914304175","https://openalex.org/W2914783728","https://openalex.org/W2914951492","https://openalex.org/W2933377349","https://openalex.org/W2953388933","https://openalex.org/W3034383590","https://openalex.org/W3121257585","https://openalex.org/W3133702157","https://openalex.org/W3203321135","https://openalex.org/W4205160672","https://openalex.org/W4205857304","https://openalex.org/W4210320781","https://openalex.org/W4221143046","https://openalex.org/W4224306499","https://openalex.org/W4224988179","https://openalex.org/W4225006216","https://openalex.org/W4225131971","https://openalex.org/W4225165463","https://openalex.org/W4226278401","https://openalex.org/W4230939871","https://openalex.org/W4236965008","https://openalex.org/W4239083664","https://openalex.org/W4249540773","https://openalex.org/W4250543126","https://openalex.org/W4253763531","https://openalex.org/W4281250694","https://openalex.org/W4281483047","https://openalex.org/W4286892945","https://openalex.org/W4287799740","https://openalex.org/W4287854593","https://openalex.org/W4293138840","https://openalex.org/W4293998609","https://openalex.org/W4297685388","https://openalex.org/W4297967137","https://openalex.org/W4303648545","https://openalex.org/W4307311873","https://openalex.org/W4307472389","https://openalex.org/W4310829078","https://openalex.org/W4313680079","https://openalex.org/W4321485561","https://openalex.org/W4323568846","https://openalex.org/W4362601804","https://openalex.org/W4367678142","https://openalex.org/W4377130677","https://openalex.org/W4377864096","https://openalex.org/W4379251563","https://openalex.org/W4379261254","https://openalex.org/W4383180610","https://openalex.org/W4383681451","https://openalex.org/W4383681480","https://openalex.org/W4385438855","https://openalex.org/W4385565170","https://openalex.org/W4385570813","https://openalex.org/W4385572830","https://openalex.org/W4385572867","https://openalex.org/W4385714498","https://openalex.org/W4385750141","https://openalex.org/W4385965410","https://openalex.org/W4385965642","https://openalex.org/W4385966440","https://openalex.org/W4385968025","https://openalex.org/W4386554921","https://openalex.org/W4387427446","https://openalex.org/W4387993417","https://openalex.org/W4388276256","https://openalex.org/W4388886073","https://openalex.org/W4390872747","https://openalex.org/W4391046141","https://openalex.org/W4391591806","https://openalex.org/W4392903699","https://openalex.org/W4392972228","https://openalex.org/W4401042726","https://openalex.org/W4403430763","https://openalex.org/W4403577657","https://openalex.org/W4404782209","https://openalex.org/W6824816265","https://openalex.org/W6851275496"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4286908577","https://openalex.org/W2886410948"],"abstract_inverted_index":{"LLM":[0,32,47,110],"chains":[1,111],"enable":[2],"complex":[3,22],"tasks":[4,23,26],"by":[5],"decomposing":[6],"work":[7],"into":[8,24],"a":[9,61,71,117,123],"sequence":[10],"of":[11,18],"subtasks.":[12],"Similarly,":[13],"the":[14,36,54,75,127],"more":[15],"established":[16],"techniques":[17,97],"crowdsourcing":[19,38,55,99,106],"workflows":[20,39,88,107],"decompose":[21],"smaller":[25],"for":[27,46,64,137,144],"human":[28,41],"crowdworkers.":[29],"Chains":[30],"address":[31,40],"errors":[33],"analogously":[34],"to":[35,59,78,91,102,108],"way":[37],"error.":[42],"To":[43,94],"characterize":[44],"opportunities":[45],"chaining,":[48,103],"we":[49,104,134],"survey":[50],"107":[51],"papers":[52],"across":[53,112],"and":[56,74,121,130,141,147],"chaining":[57],"literature":[58],"construct":[60],"design":[62,68,128,140],"space":[63,69,129],"chain":[65,139],"development.":[66,148],"The":[67],"covers":[70],"designer\u2019s":[72],"objectives":[73],"tactics":[76,90],"used":[77],"build":[79],"workflows.":[80],"We":[81],"then":[82],"surface":[83],"strategies":[84],"that":[85],"mediate":[86],"how":[87,96],"use":[89],"achieve":[92],"objectives.":[93],"explore":[95],"from":[98],"may":[100],"apply":[101],"adapt":[105],"implement":[109],"three":[113],"case":[114,132],"studies:":[115],"creating":[116],"taxonomy,":[118],"shortening":[119],"text,":[120],"writing":[122],"short":[124],"story.":[125],"From":[126],"our":[131],"studies,":[133],"identify":[135],"takeaways":[136],"effective":[138],"raise":[142],"implications":[143],"future":[145],"research":[146]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":4}],"updated_date":"2026-04-09T08:11:56.329763","created_date":"2025-10-10T00:00:00"}