{"id":"https://openalex.org/W4360978668","doi":"https://doi.org/10.1145/3581754.3584136","title":"Supporting Qualitative Analysis with Large Language Models: Combining Codebook with GPT-3 for Deductive Coding","display_name":"Supporting Qualitative Analysis with Large Language Models: Combining Codebook with GPT-3 for Deductive Coding","publication_year":2023,"publication_date":"2023-03-26","ids":{"openalex":"https://openalex.org/W4360978668","doi":"https://doi.org/10.1145/3581754.3584136"},"language":"en","primary_location":{"id":"doi:10.1145/3581754.3584136","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3581754.3584136","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"28th International Conference on Intelligent User Interfaces","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2304.10548","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5083645580","display_name":"Ziang Xiao","orcid":"https://orcid.org/0000-0003-3368-0180"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]},{"id":"https://openalex.org/I4210099137","display_name":"Research Canada","ror":"https://ror.org/0103eqz62","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210099137"]}],"countries":["CA","US"],"is_corresponding":true,"raw_author_name":"Ziang Xiao","raw_affiliation_strings":["Microsoft Research, Canada and Johns Hopkins University, United States"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, Canada and Johns Hopkins University, United States","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4210099137","https://openalex.org/I145311948"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043177488","display_name":"Xingdi Yuan","orcid":"https://orcid.org/0000-0002-7660-0059"},"institutions":[{"id":"https://openalex.org/I4210153468","display_name":"Microsoft (Canada)","ror":"https://ror.org/04xhxg104","country_code":"CA","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210153468"]},{"id":"https://openalex.org/I4402554038","display_name":"Microsoft Research Montr\u00e9al (Canada)","ror":"https://ror.org/05xdft911","country_code":null,"type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4402554038"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Xingdi Yuan","raw_affiliation_strings":["Microsoft Research, Canada","Microsoft Research Montreal (Canada)"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, Canada","institution_ids":["https://openalex.org/I4210153468"]},{"raw_affiliation_string":"Microsoft Research Montreal (Canada)","institution_ids":["https://openalex.org/I4210153468","https://openalex.org/I4402554038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024320659","display_name":"Q. Vera Liao","orcid":"https://orcid.org/0000-0003-4543-7196"},"institutions":[{"id":"https://openalex.org/I4210153468","display_name":"Microsoft (Canada)","ror":"https://ror.org/04xhxg104","country_code":"CA","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210153468"]},{"id":"https://openalex.org/I4402554038","display_name":"Microsoft Research Montr\u00e9al (Canada)","ror":"https://ror.org/05xdft911","country_code":null,"type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4402554038"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Q. Vera Liao","raw_affiliation_strings":["Microsoft Research, Canada","Microsoft Research Montreal (Canada)"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, Canada","institution_ids":["https://openalex.org/I4210153468"]},{"raw_affiliation_string":"Microsoft Research Montreal (Canada)","institution_ids":["https://openalex.org/I4210153468","https://openalex.org/I4402554038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009797437","display_name":"Rania Abdelghani","orcid":"https://orcid.org/0000-0002-6361-6609"},"institutions":[{"id":"https://openalex.org/I1326498283","display_name":"Institut national de recherche en sciences et technologies du num\u00e9rique","ror":"https://ror.org/02kvxyf05","country_code":"FR","type":"government","lineage":["https://openalex.org/I1326498283"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Rania Abdelghani","raw_affiliation_strings":["Inria, France","Flowers - Flowing Epigenetic Robots and Systems (200, avenue de la Vieille Tour \r\n33405 Talence cedex - France)"],"affiliations":[{"raw_affiliation_string":"Inria, France","institution_ids":["https://openalex.org/I1326498283"]},{"raw_affiliation_string":"Flowers - Flowing Epigenetic Robots and Systems (200, avenue de la Vieille Tour \r\n33405 Talence cedex - France)","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5085903884","display_name":"Pierre-Yves Oudeyer","orcid":"https://orcid.org/0000-0002-1277-130X"},"institutions":[{"id":"https://openalex.org/I1326498283","display_name":"Institut national de recherche en sciences et technologies du num\u00e9rique","ror":"https://ror.org/02kvxyf05","country_code":"FR","type":"government","lineage":["https://openalex.org/I1326498283"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Pierre-Yves Oudeyer","raw_affiliation_strings":["Inria, France","Flowers - Flowing Epigenetic Robots and Systems (200, avenue de la Vieille Tour \r\n33405 Talence cedex - France)"],"affiliations":[{"raw_affiliation_string":"Inria, France","institution_ids":["https://openalex.org/I1326498283"]},{"raw_affiliation_string":"Flowers - Flowing Epigenetic Robots and Systems (200, avenue de la Vieille Tour \r\n33405 Talence cedex - France)","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5083645580"],"corresponding_institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I145311948","https://openalex.org/I4210099137"],"apc_list":null,"apc_paid":null,"fwci":32.8905,"has_fulltext":true,"cited_by_count":191,"citation_normalized_percentile":{"value":0.99816947,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"75","last_page":"78"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.9937000274658203,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9767000079154968,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/codebook","display_name":"Codebook","score":0.8156088590621948},{"id":"https://openalex.org/keywords/generalizability-theory","display_name":"Generalizability theory","score":0.7419017553329468},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7380708456039429},{"id":"https://openalex.org/keywords/coding","display_name":"Coding (social sciences)","score":0.7227516174316406},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.4857807457447052},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4776279330253601},{"id":"https://openalex.org/keywords/curiosity","display_name":"Curiosity","score":0.4773523807525635},{"id":"https://openalex.org/keywords/qualitative-analysis","display_name":"Qualitative analysis","score":0.46784886717796326},{"id":"https://openalex.org/keywords/qualitative-research","display_name":"Qualitative research","score":0.4660654366016388},{"id":"https://openalex.org/keywords/task-analysis","display_name":"Task analysis","score":0.4528069496154785},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4410209655761719},{"id":"https://openalex.org/keywords/qualitative-property","display_name":"Qualitative property","score":0.43636730313301086},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.40898334980010986},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3622381091117859},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.16827791929244995},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.104188472032547}],"concepts":[{"id":"https://openalex.org/C127759330","wikidata":"https://www.wikidata.org/wiki/Q637416","display_name":"Codebook","level":2,"score":0.8156088590621948},{"id":"https://openalex.org/C27158222","wikidata":"https://www.wikidata.org/wiki/Q5532422","display_name":"Generalizability theory","level":2,"score":0.7419017553329468},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7380708456039429},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.7227516174316406},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4857807457447052},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4776279330253601},{"id":"https://openalex.org/C33435437","wikidata":"https://www.wikidata.org/wiki/Q366791","display_name":"Curiosity","level":2,"score":0.4773523807525635},{"id":"https://openalex.org/C3018587665","wikidata":"https://www.wikidata.org/wiki/Q7268696","display_name":"Qualitative analysis","level":3,"score":0.46784886717796326},{"id":"https://openalex.org/C190248442","wikidata":"https://www.wikidata.org/wiki/Q839486","display_name":"Qualitative research","level":2,"score":0.4660654366016388},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.4528069496154785},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4410209655761719},{"id":"https://openalex.org/C87156501","wikidata":"https://www.wikidata.org/wiki/Q7268708","display_name":"Qualitative property","level":2,"score":0.43636730313301086},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.40898334980010986},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3622381091117859},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.16827791929244995},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.104188472032547},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C138496976","wikidata":"https://www.wikidata.org/wiki/Q175002","display_name":"Developmental psychology","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3581754.3584136","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3581754.3584136","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"28th International Conference on Intelligent User Interfaces","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2304.10548","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2304.10548","pdf_url":"https://arxiv.org/pdf/2304.10548","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:HAL:hal-04369097v1","is_oa":true,"landing_page_url":"https://inria.hal.science/hal-04369097","pdf_url":null,"source":{"id":"https://openalex.org/S4406922461","display_name":"SPIRE - Sciences Po Institutional REpository","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IUI 2023 - 28th International Conference on Intelligent User Interfaces, Mar 2023, Sydney, Australia. pp.75-78, ⟨10.1145/3581754.3584136⟩","raw_type":"Conference papers"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2304.10548","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2304.10548","pdf_url":"https://arxiv.org/pdf/2304.10548","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/8","display_name":"Decent work and economic growth","score":0.5699999928474426}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4360978668.pdf","grobid_xml":"https://content.openalex.org/works/W4360978668.grobid-xml"},"referenced_works_count":18,"referenced_works":["https://openalex.org/W387031967","https://openalex.org/W2142225512","https://openalex.org/W2252208617","https://openalex.org/W2546996098","https://openalex.org/W2588117145","https://openalex.org/W3124026294","https://openalex.org/W3164718925","https://openalex.org/W3173777717","https://openalex.org/W4221055872","https://openalex.org/W4224308101","https://openalex.org/W4226071117","https://openalex.org/W4229005866","https://openalex.org/W4238846128","https://openalex.org/W4286252734","https://openalex.org/W4292779060","https://openalex.org/W4294214983","https://openalex.org/W4296965999","https://openalex.org/W4297801719"],"related_works":["https://openalex.org/W2293149949","https://openalex.org/W2026099691","https://openalex.org/W4284672201","https://openalex.org/W2377486419","https://openalex.org/W2943202426","https://openalex.org/W2736714427","https://openalex.org/W2163679795","https://openalex.org/W2137816434","https://openalex.org/W2017956276","https://openalex.org/W2048606991"],"abstract_inverted_index":{"Qualitative":[0],"analysis":[1,77],"of":[2,52,63,75,91,94],"textual":[3],"contents":[4],"unpacks":[5],"rich":[6],"and":[7,42,146,155],"valuable":[8],"information":[9],"by":[10,48,125],"assigning":[11],"labels":[12],"to":[13,83,136,151],"the":[14,49,61,85],"data.":[15],"However,":[16],"this":[17,57],"process":[18],"is":[19],"often":[20],"labor-intensive,":[21],"particularly":[22],"when":[23],"working":[24],"with":[25,128,139],"large":[26,64],"datasets.":[27],"While":[28],"recent":[29],"AI-based":[30],"tools":[31],"demonstrate":[32],"utility,":[33],"researchers":[34,79],"may":[35],"not":[36],"have":[37],"readily":[38],"available":[39],"AI":[40],"resources":[41],"expertise,":[43],"let":[44],"alone":[45],"be":[46,102],"challenged":[47],"limited":[50],"generalizability":[51],"those":[53],"task-specific":[54,96],"models.":[55],"In":[56],"study,":[58,122],"we":[59,123],"explored":[60],"use":[62,80],"language":[65],"models":[66],"(LLMs)":[67],"in":[68,148],"supporting":[69],"deductive":[70],"coding,":[71],"a":[72,88,98,114,120],"major":[73],"category":[74],"qualitative":[76,153],"where":[78],"pre-determined":[81],"codebooks":[82],"label":[84],"data":[86],"into":[87],"fixed":[89],"set":[90],"codes.":[92],"Instead":[93],"training":[95],"models,":[97],"pre-trained":[99],"LLM":[100],"could":[101],"used":[103],"directly":[104],"for":[105],"various":[106],"tasks":[107],"without":[108],"fine-tuning":[109],"through":[110],"prompt":[111],"learning.":[112],"Using":[113],"curiosity-driven":[115],"questions":[116],"coding":[117,154],"task":[118],"as":[119],"case":[121],"found,":[124],"combining":[126],"GPT-3":[127],"expert-drafted":[129],"codebooks,":[130],"our":[131],"proposed":[132],"approach":[133],"achieved":[134],"fair":[135],"substantial":[137],"agreements":[138],"expert-coded":[140],"results.":[141],"We":[142],"lay":[143],"out":[144],"challenges":[145],"opportunities":[147],"using":[149],"LLMs":[150],"support":[152],"beyond.":[156]},"counts_by_year":[{"year":2026,"cited_by_count":26},{"year":2025,"cited_by_count":78},{"year":2024,"cited_by_count":72},{"year":2023,"cited_by_count":15}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2023-03-30T00:00:00"}