{"id":"https://openalex.org/W7131311596","doi":"https://doi.org/10.1145/3794763.3798172","title":"LLMs for Qualitative Data Analysis Fail on Security-specific Comments in Human Experiments","display_name":"LLMs for Qualitative Data Analysis Fail on Security-specific Comments in Human Experiments","publication_year":2026,"publication_date":"2026-04-12","ids":{"openalex":"https://openalex.org/W7131311596","doi":"https://doi.org/10.1145/3794763.3798172"},"language":null,"primary_location":{"id":"doi:10.1145/3794763.3798172","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3794763.3798172","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 34th IEEE/ACM International Conference on Program Comprehension","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3794763.3798172","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5116211301","display_name":"Maria Camporese","orcid":null},"institutions":[{"id":"https://openalex.org/I193223587","display_name":"University of Trento","ror":"https://ror.org/05trd4x28","country_code":"IT","type":"education","lineage":["https://openalex.org/I193223587"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Maria Camporese","raw_affiliation_strings":["University of Trento, Trento, Italy"],"raw_orcid":"https://orcid.org/0009-0009-1178-0210","affiliations":[{"raw_affiliation_string":"University of Trento, Trento, Italy","institution_ids":["https://openalex.org/I193223587"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126715054","display_name":"Fabio Massacci","orcid":null},"institutions":[{"id":"https://openalex.org/I193223587","display_name":"University of Trento","ror":"https://ror.org/05trd4x28","country_code":"IT","type":"education","lineage":["https://openalex.org/I193223587"]},{"id":"https://openalex.org/I865915315","display_name":"Vrije Universiteit Amsterdam","ror":"https://ror.org/008xxew50","country_code":"NL","type":"education","lineage":["https://openalex.org/I865915315"]}],"countries":["IT","NL"],"is_corresponding":false,"raw_author_name":"Fabio Massacci","raw_affiliation_strings":["University of Trento, Trento, Italy and Vrije Universiteit Amsterdam, Amsterdam, Netherlands"],"raw_orcid":"https://orcid.org/0000-0002-1091-8486","affiliations":[{"raw_affiliation_string":"University of Trento, Trento, Italy and Vrije Universiteit Amsterdam, Amsterdam, Netherlands","institution_ids":["https://openalex.org/I193223587","https://openalex.org/I865915315"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5062483625","display_name":"Yuanjun Gong","orcid":"https://orcid.org/0000-0002-4661-904X"},"institutions":[{"id":"https://openalex.org/I193223587","display_name":"University of Trento","ror":"https://ror.org/05trd4x28","country_code":"IT","type":"education","lineage":["https://openalex.org/I193223587"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Yuanjun Gong","raw_affiliation_strings":["University of Trento, Trento, Italy"],"raw_orcid":"https://orcid.org/0000-0002-4661-904X","affiliations":[{"raw_affiliation_string":"University of Trento, Trento, Italy","institution_ids":["https://openalex.org/I193223587"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"515","last_page":"526"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":null,"topics":[],"keywords":[{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5703999996185303},{"id":"https://openalex.org/keywords/qualitative-research","display_name":"Qualitative research","score":0.4235000014305115},{"id":"https://openalex.org/keywords/qualitative-analysis","display_name":"Qualitative analysis","score":0.3709999918937683},{"id":"https://openalex.org/keywords/qualitative-property","display_name":"Qualitative property","score":0.3409999907016754},{"id":"https://openalex.org/keywords/data-collection","display_name":"Data collection","score":0.274399995803833}],"concepts":[{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5703999996185303},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5611000061035156},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4440000057220459},{"id":"https://openalex.org/C190248442","wikidata":"https://www.wikidata.org/wiki/Q839486","display_name":"Qualitative research","level":2,"score":0.4235000014305115},{"id":"https://openalex.org/C3018587665","wikidata":"https://www.wikidata.org/wiki/Q7268696","display_name":"Qualitative analysis","level":3,"score":0.3709999918937683},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3434000015258789},{"id":"https://openalex.org/C87156501","wikidata":"https://www.wikidata.org/wiki/Q7268708","display_name":"Qualitative property","level":2,"score":0.3409999907016754},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.33880001306533813},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.28299999237060547},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.28290000557899475},{"id":"https://openalex.org/C162118730","wikidata":"https://www.wikidata.org/wiki/Q1128453","display_name":"Actuarial science","level":1,"score":0.2786000072956085},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.274399995803833},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.2635999917984009},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.25440001487731934}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3794763.3798172","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3794763.3798172","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 34th IEEE/ACM International Conference on Program Comprehension","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3794763.3798172","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3794763.3798172","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 34th IEEE/ACM International Conference on Program Comprehension","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"[Background:]":[0],"Thematic":[1],"analysis":[2],"of":[3],"free-text":[4,84],"justifications":[5],"in":[6,83],"human":[7,33,68,87,98,149],"experiments":[8],"provides":[9],"significant":[10],"qualitative":[11],"insights.":[12],"Yet,":[13],"it":[14],"is":[15],"costly":[16],"because":[17],"reliable":[18],"annotations":[19],"require":[20,47],"multiple":[21],"domain":[22],"experts.":[23],"Large":[24],"language":[25],"models":[26],"(LLMs)":[27],"seem":[28],"ideal":[29],"candidates":[30],"to":[31,78,96,145],"replace":[32,147],"annotators.":[34],"[Problem:]":[35],"Coding":[36],"security-specific":[37],"aspects":[38],"(code":[39],"identifiers":[40],"mentioned,":[41],"lines-of-code-mentioned,":[42],"security":[43,65],"keywords":[44],"mentioned)":[45],"may":[46],"deeper":[48],"contextual":[49],"understanding":[50],"than":[51],"sentiment":[52],"classification.":[53],"[Objective:]":[54],"explore":[55],"whether":[56],"LLMs":[57,75,156],"can":[58],"act":[59],"as":[60],"automated":[61],"annotators":[62,99],"for":[63],"technical":[64],"comments":[66,85],"by":[67,86],"subjects.":[69],"[Method:]":[70],"We":[71,106,126],"prompt":[72],"four":[73],"best":[74,112],"on":[76],"LiveBench":[77],"detect":[79],"nine":[80],"security-relevant":[81],"codes":[82,141],"subjects":[88],"analyzing":[89],"vulnerable":[90],"code":[91,133],"snippets.":[92],"Outputs":[93],"are":[94,137,160],"compared":[95],"the":[97,132],"along":[100],"with":[101,119,131,154],"Cohen\u2019s":[102],"Kappa":[103],"(chance-corrected":[104],"accuracy).":[105],"test":[107],"different":[108],"prompts":[109],"mimicking":[110],"annotation":[111,158],"practices:":[113],"emerging":[114],"codes,":[115],"a":[116,148],"detailed":[117],"codebook":[118],"examples,":[120],"and":[121,142,157],"conflicting":[122],"examples.":[123],"[Negative":[124],"Results:]":[125],"observed":[127],"marked":[128],"improvements":[129],"only":[130],"descriptions,":[134],"but":[135],"they":[136],"not":[138,143],"uniform":[139],"across":[140],"sufficient":[144],"reliably":[146],"annotator.":[150],"[Limitations:]":[151],"Additional":[152],"studies":[153],"more":[155],"tasks":[159],"needed.":[161]},"counts_by_year":[],"updated_date":"2026-07-30T17:31:21.811387","created_date":"2026-02-25T00:00:00"}