{"id":"https://openalex.org/W4411120979","doi":"https://doi.org/10.1609/icwsm.v19i1.35807","title":"What's in a Prompt?: A Large-Scale Experiment to Assess the Impact of Prompt Design on the Compliance and Accuracy of LLM-Generated Text Annotations","display_name":"What's in a Prompt?: A Large-Scale Experiment to Assess the Impact of Prompt Design on the Compliance and Accuracy of LLM-Generated Text Annotations","publication_year":2025,"publication_date":"2025-06-07","ids":{"openalex":"https://openalex.org/W4411120979","doi":"https://doi.org/10.1609/icwsm.v19i1.35807"},"language":"en","primary_location":{"id":"doi:10.1609/icwsm.v19i1.35807","is_oa":true,"landing_page_url":"https://doi.org/10.1609/icwsm.v19i1.35807","pdf_url":"https://ojs.aaai.org/index.php/ICWSM/article/download/35807/37961","source":{"id":"https://openalex.org/S4387284482","display_name":"Proceedings of the International AAAI Conference on Web and Social Media","issn_l":"2162-3449","issn":["2162-3449","2334-0770"],"is_oa":true,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the International AAAI Conference on Web and Social Media","raw_type":"journal-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://ojs.aaai.org/index.php/ICWSM/article/download/35807/37961","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5076021360","display_name":"Shubham Atreja","orcid":"https://orcid.org/0000-0002-0056-3060"},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shubham Atreja","raw_affiliation_strings":["University of Michigan School of Information"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Michigan School of Information","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048428275","display_name":"Joshua Ashkinaze","orcid":"https://orcid.org/0000-0002-6206-3109"},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Joshua Ashkinaze","raw_affiliation_strings":["University of Michigan School of Information"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Michigan School of Information","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031522503","display_name":"Lingyao Li","orcid":"https://orcid.org/0000-0001-5888-8311"},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lingyao Li","raw_affiliation_strings":["University of Michigan School of Information"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Michigan School of Information","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038833789","display_name":"Julia Mendelsohn","orcid":"https://orcid.org/0000-0002-8584-2578"},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Julia Mendelsohn","raw_affiliation_strings":["University of Michigan School of Information"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Michigan School of Information","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5077179813","display_name":"Libby Hemphill","orcid":"https://orcid.org/0000-0002-3793-7281"},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Libby Hemphill","raw_affiliation_strings":["University of Michigan School of Information"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Michigan School of Information","institution_ids":["https://openalex.org/I27837315"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I27837315"],"apc_list":null,"apc_paid":null,"fwci":19.6422,"has_fulltext":true,"cited_by_count":14,"citation_normalized_percentile":{"value":0.99520642,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":"19","issue":null,"first_page":"122","last_page":"145"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9955000281333923,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9955000281333923,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.9478999972343445,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.9401000142097473,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/compliance","display_name":"Compliance (psychology)","score":0.6517094969749451},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.5565323233604431},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.48333093523979187},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.24640986323356628},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.12113699316978455},{"id":"https://openalex.org/keywords/cartography","display_name":"Cartography","score":0.07777604460716248},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.057523101568222046}],"concepts":[{"id":"https://openalex.org/C2781460075","wikidata":"https://www.wikidata.org/wiki/Q1399332","display_name":"Compliance (psychology)","level":2,"score":0.6517094969749451},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.5565323233604431},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.48333093523979187},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.24640986323356628},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.12113699316978455},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.07777604460716248},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.057523101568222046}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1609/icwsm.v19i1.35807","is_oa":true,"landing_page_url":"https://doi.org/10.1609/icwsm.v19i1.35807","pdf_url":"https://ojs.aaai.org/index.php/ICWSM/article/download/35807/37961","source":{"id":"https://openalex.org/S4387284482","display_name":"Proceedings of the International AAAI Conference on Web and Social Media","issn_l":"2162-3449","issn":["2162-3449","2334-0770"],"is_oa":true,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the International AAAI Conference on Web and Social Media","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1609/icwsm.v19i1.35807","is_oa":true,"landing_page_url":"https://doi.org/10.1609/icwsm.v19i1.35807","pdf_url":"https://ojs.aaai.org/index.php/ICWSM/article/download/35807/37961","source":{"id":"https://openalex.org/S4387284482","display_name":"Proceedings of the International AAAI Conference on Web and Social Media","issn_l":"2162-3449","issn":["2162-3449","2334-0770"],"is_oa":true,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the International AAAI Conference on Web and Social Media","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2781662787","display_name":"FW-HTF-RM: Collaborative Research: Augmenting Social Media Content Moderation","funder_award_id":"1928434","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4411120979.pdf","grobid_xml":"https://content.openalex.org/works/W4411120979.grobid-xml"},"referenced_works_count":26,"referenced_works":["https://openalex.org/W2251939518","https://openalex.org/W2954444514","https://openalex.org/W2986889260","https://openalex.org/W3087316519","https://openalex.org/W4283168936","https://openalex.org/W4306808908","https://openalex.org/W4321175700","https://openalex.org/W4321276802","https://openalex.org/W4327525501","https://openalex.org/W4361193900","https://openalex.org/W4362597995","https://openalex.org/W4363676214","https://openalex.org/W4366850809","https://openalex.org/W4377130677","https://openalex.org/W4379598302","https://openalex.org/W4387113760","https://openalex.org/W4387725311","https://openalex.org/W4389500600","https://openalex.org/W4389524061","https://openalex.org/W4411630221","https://openalex.org/W6734394377","https://openalex.org/W6749630143","https://openalex.org/W6753376294","https://openalex.org/W6799802702","https://openalex.org/W6860002659","https://openalex.org/W6966334233"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2410395228","https://openalex.org/W2390279801","https://openalex.org/W3125941065","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2484615095"],"abstract_inverted_index":{"Manually":[0],"annotating":[1],"data":[2],"for":[3,105,140,179],"computational":[4],"social":[5],"science":[6],"tasks":[7,25,84,131],"can":[8,21,119,143],"be":[9],"costly,":[10],"time-consuming,":[11],"and":[12,39,55,57,66,72,81,89,98,115,163,176],"emotionally":[13],"draining.":[14],"While":[15],"recent":[16],"work":[17,169],"suggests":[18],"that":[19,95],"LLMs":[20,181],"perform":[22],"such":[23],"annotation":[24],"in":[26,147,182],"zero-shot":[27],"settings,":[28],"little":[29],"is":[30],"known":[31],"about":[32],"how":[33,49],"prompt":[34,58,67,136,158],"design":[35,59,159],"impacts":[36],"LLMs'":[37,113],"compliance":[38,71,97,114],"accuracy.":[40,116],"We":[41],"conduct":[42],"a":[43,173,177],"large-scale":[44],"multi-prompt":[45],"experiment":[46],"to":[47,127],"test":[48],"model":[50],"selection":[51],"(GPT-4o,":[52],"GPT-3.5,":[53],"PaLM2,":[54],"Falcon7b)":[56],"features":[60],"(definition":[61],"inclusion,":[62],"output":[63],"type,":[64],"explanation,":[65],"length)":[68],"impact":[69,156],"the":[70,148,155,161],"accuracy":[73,99,129],"of":[74,109,150,157,165],"LLM-generated":[75,151,166],"annotations":[76],"on":[77,130,160],"four":[78],"highly":[79],"relevant":[80],"diverse":[82],"CSS":[83,183],"(toxicity,":[85],"sentiment,":[86],"rumor":[87],"stance,":[88],"news":[90],"frames).":[91],"Our":[92],"results":[93],"show":[94],"LLM":[96],"are":[100],"prompt-dependent.":[101],"For":[102],"instance,":[103],"prompting":[104,122],"numerical":[106],"scores":[107],"instead":[108],"labels":[110],"reduces":[111],"all":[112],"Concise":[117],"prompts":[118],"significantly":[120],"reduce":[121],"costs":[123],"but":[124],"also":[125],"lead":[126],"lower":[128],"like":[132,138],"toxicity.":[133],"Furthermore,":[134],"minor":[135],"changes":[137,146],"asking":[139],"an":[141],"explanation":[142],"cause":[144],"large":[145],"distribution":[149,164],"labels.":[152],"By":[153],"assessing":[154],"quality":[162],"annotations,":[167],"this":[168],"serves":[170],"as":[171],"both":[172],"practical":[174],"guide":[175],"warning":[178],"using":[180],"research.":[184]},"counts_by_year":[{"year":2026,"cited_by_count":9},{"year":2025,"cited_by_count":5}],"updated_date":"2026-08-24T07:32:12.397491","created_date":"2025-06-08T00:00:00"}