{"id":"https://openalex.org/W4385014863","doi":"https://doi.org/10.48550/arxiv.2307.10168","title":"LLMs as Workers in Human-Computational Algorithms? Replicating Crowdsourcing Pipelines with LLMs","display_name":"LLMs as Workers in Human-Computational Algorithms? Replicating Crowdsourcing Pipelines with LLMs","publication_year":2023,"publication_date":"2023-07-19","ids":{"openalex":"https://openalex.org/W4385014863","doi":"https://doi.org/10.48550/arxiv.2307.10168"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2307.10168","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2307.10168","pdf_url":"https://arxiv.org/pdf/2307.10168","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2307.10168","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5004225142","display_name":"Tongshuang Wu","orcid":"https://orcid.org/0000-0003-1630-0588"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Tongshuang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051842323","display_name":"Haiyi Zhu","orcid":"https://orcid.org/0000-0001-7271-9100"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, Haiyi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092513653","display_name":"Maya Albayrak","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Albayrak, Maya","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114068490","display_name":"Alexis Axon","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Axon, Alexis","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013007353","display_name":"Amanda Bertsch","orcid":"https://orcid.org/0000-0002-1368-1111"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bertsch, Amanda","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111192886","display_name":"Wenxing Deng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Deng, Wenxing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101317643","display_name":"Ziqi Ding","orcid":"https://orcid.org/0009-0002-9971-0899"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ding, Ziqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092513654","display_name":"Bill Guo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guo, Bill","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108855136","display_name":"Sireesh Gururaja","orcid":"https://orcid.org/0009-0004-0309-3393"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gururaja, Sireesh","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056050477","display_name":"Tzu-Sheng Kuo","orcid":"https://orcid.org/0000-0002-1504-7640"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kuo, Tzu-Sheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006849693","display_name":"Jenny T. Liang","orcid":"https://orcid.org/0000-0001-6722-9959"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liang, Jenny T.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061907283","display_name":"Ryan Wen Liu","orcid":"https://orcid.org/0000-0002-1591-5583"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Ryan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092513648","display_name":"Ihita Mandal","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mandal, Ihita","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000331140","display_name":"Jeremiah Milbauer","orcid":"https://orcid.org/0000-0003-2809-534X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Milbauer, Jeremiah","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112733575","display_name":"Xiaolin Ni","orcid":"https://orcid.org/0009-0009-2660-9539"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ni, Xiaolin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092513649","display_name":"Namrata Padmanabhan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Padmanabhan, Namrata","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092513650","display_name":"Subhashini Ramkumar","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ramkumar, Subhashini","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092513651","display_name":"Alexis Sudjianto","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sudjianto, Alexis","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112964295","display_name":"Jordan Taylor","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Taylor, Jordan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102912419","display_name":"Ying-Jui Tseng","orcid":"https://orcid.org/0009-0006-1801-6061"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tseng, Ying-Jui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092513652","display_name":"Patricia Vaidos","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Vaidos, Patricia","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009461496","display_name":"Zhijin Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Zhijin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102536970","display_name":"Wei Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Wei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5101966381","display_name":"Chenyang Yang","orcid":"https://orcid.org/0000-0001-5016-7296"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Chenyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":9,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12784","display_name":"Modular Robots and Swarm Intelligence","score":0.9927999973297119,"subfield":{"id":"https://openalex.org/subfields/2210","display_name":"Mechanical Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9775999784469604,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.8525656461715698},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.38311219215393066},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.349606454372406},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.09857967495918274}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8525656461715698},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.38311219215393066},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.349606454372406},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.09857967495918274}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2307.10168","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2307.10168","pdf_url":"https://arxiv.org/pdf/2307.10168","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2307.10168","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2307.10168","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article-journal"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2307.10168","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2307.10168","pdf_url":"https://arxiv.org/pdf/2307.10168","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4385014863.pdf","grobid_xml":"https://content.openalex.org/works/W4385014863.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4286908577","https://openalex.org/W2886410948","https://openalex.org/W2025875869"],"abstract_inverted_index":{"LLMs":[0,33,44,113],"have":[1],"shown":[2],"promise":[3],"in":[4,8,51,149],"replicating":[5,122],"human-like":[6],"behavior":[7],"crowdsourcing":[9,38,123],"tasks":[10,138,159],"that":[11,42,121],"were":[12],"previously":[13],"thought":[14],"to":[15,18,94,129,163],"be":[16],"exclusive":[17],"human":[19,89],"abilities.":[20],"However,":[21],"current":[22],"efforts":[23],"focus":[24],"mainly":[25],"on":[26,88,136,143],"simple":[27],"atomic":[28],"tasks.":[29],"We":[30,40,86],"explore":[31],"whether":[32],"can":[34,45,154],"replicate":[35],"more":[36],"complex":[37,150],"pipelines.":[39],"find":[41],"modern":[43],"simulate":[46],"some":[47],"of":[48,59,68,99,109,157],"crowdworkers'":[49],"abilities":[50],"these":[52,84],"``human":[53],"computation":[54],"algorithms,''":[55],"but":[56],"the":[57,71,78,97,107,132,158],"level":[58],"success":[60],"is":[61],"variable":[62],"and":[63,77,90,105,112,145],"influenced":[64],"by":[65],"requesters'":[66],"understanding":[67],"LLM":[69,134],"capabilities,":[70],"specific":[72],"skills":[73],"required":[74],"for":[75,82,103],"sub-tasks,":[76],"optimal":[79],"interaction":[80],"modality":[81],"performing":[83],"sub-tasks.":[85],"reflect":[87],"LLMs'":[91,147],"different":[92,137],"sensitivities":[93],"instructions,":[95],"stress":[96],"importance":[98],"enabling":[100],"human-facing":[101],"safeguards":[102],"LLMs,":[104],"discuss":[106],"potential":[108,148],"training":[110],"humans":[111],"with":[114],"complementary":[115],"skill":[116],"sets.":[117],"Crucially,":[118],"we":[119],"show":[120],"pipelines":[124],"offers":[125],"a":[126],"valuable":[127],"platform":[128],"investigate":[130],"1)":[131],"relative":[133],"strengths":[135],"(by":[139],"cross-comparing":[140],"their":[141],"performances":[142],"sub-tasks)":[144],"2)":[146],"tasks,":[151],"where":[152],"they":[153],"complete":[155],"part":[156],"while":[160],"leaving":[161],"others":[162],"humans.":[164]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}