{"id":"https://openalex.org/W4406738283","doi":"https://doi.org/10.1145/3715007","title":"An Empirical Study on Challenges for LLM Application Developers","display_name":"An Empirical Study on Challenges for LLM Application Developers","publication_year":2025,"publication_date":"2025-01-23","ids":{"openalex":"https://openalex.org/W4406738283","doi":"https://doi.org/10.1145/3715007"},"language":"en","primary_location":{"id":"doi:10.1145/3715007","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3715007","pdf_url":null,"source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100441911","display_name":"Xiang Chen","orcid":"https://orcid.org/0000-0002-1180-3891"},"institutions":[{"id":"https://openalex.org/I199305430","display_name":"Nantong University","ror":"https://ror.org/02afcvw97","country_code":"CN","type":"education","lineage":["https://openalex.org/I199305430"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Xiang Chen","raw_affiliation_strings":["Nantong University, Nantong, China","School of Artificial Intelligence and Computer Science, Nantong University, China"],"affiliations":[{"raw_affiliation_string":"Nantong University, Nantong, China","institution_ids":["https://openalex.org/I199305430"]},{"raw_affiliation_string":"School of Artificial Intelligence and Computer Science, Nantong University, China","institution_ids":["https://openalex.org/I199305430"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010878830","display_name":"Chaoyang Gao","orcid":null},"institutions":[{"id":"https://openalex.org/I199305430","display_name":"Nantong University","ror":"https://ror.org/02afcvw97","country_code":"CN","type":"education","lineage":["https://openalex.org/I199305430"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chaoyang Gao","raw_affiliation_strings":["Nantong University, Nantong, China","School of Artificial Intelligence and Computer Science, Nantong University, China"],"affiliations":[{"raw_affiliation_string":"Nantong University, Nantong, China","institution_ids":["https://openalex.org/I199305430"]},{"raw_affiliation_string":"School of Artificial Intelligence and Computer Science, Nantong University, China","institution_ids":["https://openalex.org/I199305430"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075639297","display_name":"Chunyang Chen","orcid":"https://orcid.org/0000-0003-2011-9618"},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Chunyang Chen","raw_affiliation_strings":["Technical University of Munich, Munchen, Germany","Department of Computer Science, Technical University of Munich, Germany"],"affiliations":[{"raw_affiliation_string":"Technical University of Munich, Munchen, Germany","institution_ids":["https://openalex.org/I62916508"]},{"raw_affiliation_string":"Department of Computer Science, Technical University of Munich, Germany","institution_ids":["https://openalex.org/I62916508"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051713835","display_name":"Guangbei Zhang","orcid":null},"institutions":[{"id":"https://openalex.org/I199305430","display_name":"Nantong University","ror":"https://ror.org/02afcvw97","country_code":"CN","type":"education","lineage":["https://openalex.org/I199305430"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Guangbei Zhang","raw_affiliation_strings":["Nantong University, Nantong, China","School of Artificial Intelligence and Computer Science, Nantong University, China"],"affiliations":[{"raw_affiliation_string":"Nantong University, Nantong, China","institution_ids":["https://openalex.org/I199305430"]},{"raw_affiliation_string":"School of Artificial Intelligence and Computer Science, Nantong University, China","institution_ids":["https://openalex.org/I199305430"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101595134","display_name":"Yong Liu","orcid":"https://orcid.org/0000-0003-1754-3039"},"institutions":[{"id":"https://openalex.org/I75390827","display_name":"Beijing University of Chemical Technology","ror":"https://ror.org/00df5yc52","country_code":"CN","type":"education","lineage":["https://openalex.org/I75390827"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yong Liu","raw_affiliation_strings":["Beijing University of Chemical Technology, Beijing, China","College of Information Science and Technology, Beijing University of Chemical Technology, China"],"affiliations":[{"raw_affiliation_string":"Beijing University of Chemical Technology, Beijing, China","institution_ids":["https://openalex.org/I75390827"]},{"raw_affiliation_string":"College of Information Science and Technology, Beijing University of Chemical Technology, China","institution_ids":["https://openalex.org/I75390827"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5100441911"],"corresponding_institution_ids":["https://openalex.org/I199305430"],"apc_list":null,"apc_paid":null,"fwci":48.6922,"has_fulltext":false,"cited_by_count":15,"citation_normalized_percentile":{"value":0.9979586,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"34","issue":"7","first_page":"1","last_page":"37"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13999","display_name":"Digital Rights Management and Security","score":0.988099992275238,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13999","display_name":"Digital Rights Management and Security","score":0.988099992275238,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10679","display_name":"Service-Oriented Architecture and Web Services","score":0.9642000198364258,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9373000264167786,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7512140274047852},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.35919293761253357},{"id":"https://openalex.org/keywords/engineering-management","display_name":"Engineering management","score":0.3217313289642334},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.06960496306419373}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7512140274047852},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.35919293761253357},{"id":"https://openalex.org/C110354214","wikidata":"https://www.wikidata.org/wiki/Q6314146","display_name":"Engineering management","level":1,"score":0.3217313289642334},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.06960496306419373}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3715007","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3715007","pdf_url":null,"source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1121271761","display_name":null,"funder_award_id":"Program","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G2087396116","display_name":null,"funder_award_id":"China","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3317480652","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G430625229","display_name":null,"funder_award_id":"202006","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4833460672","display_name":null,"funder_award_id":"61202006","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5994120800","display_name":null,"funder_award_id":"Natural","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6025513136","display_name":null,"funder_award_id":"1202006","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320321605","display_name":"Government of Jiangsu Province","ror":"https://ror.org/004svx814"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":55,"referenced_works":["https://openalex.org/W1991613282","https://openalex.org/W2053154970","https://openalex.org/W2087419855","https://openalex.org/W2093400716","https://openalex.org/W2104577574","https://openalex.org/W2164777277","https://openalex.org/W2395092677","https://openalex.org/W2511548333","https://openalex.org/W2850992922","https://openalex.org/W2895297154","https://openalex.org/W2955439455","https://openalex.org/W2962771808","https://openalex.org/W2968594320","https://openalex.org/W3005940936","https://openalex.org/W3027879771","https://openalex.org/W3090643686","https://openalex.org/W3091658513","https://openalex.org/W3099333834","https://openalex.org/W3100925971","https://openalex.org/W3195255266","https://openalex.org/W3196239222","https://openalex.org/W4221143046","https://openalex.org/W4307926351","https://openalex.org/W4308643152","https://openalex.org/W4320854935","https://openalex.org/W4327594616","https://openalex.org/W4367860052","https://openalex.org/W4377130677","https://openalex.org/W4384345684","https://openalex.org/W4386724197","https://openalex.org/W4387425757","https://openalex.org/W4388196756","https://openalex.org/W4388483444","https://openalex.org/W4388483497","https://openalex.org/W4389104713","https://openalex.org/W4389158474","https://openalex.org/W4389162170","https://openalex.org/W4390813110","https://openalex.org/W4390976306","https://openalex.org/W4391282616","https://openalex.org/W4391558363","https://openalex.org/W4391558545","https://openalex.org/W4391558635","https://openalex.org/W4391974543","https://openalex.org/W4393029443","https://openalex.org/W4394745253","https://openalex.org/W4399686980","https://openalex.org/W4400277330","https://openalex.org/W4400582613","https://openalex.org/W4400582623","https://openalex.org/W4400690920","https://openalex.org/W4402533615","https://openalex.org/W4402665833","https://openalex.org/W4403007658","https://openalex.org/W6838865847"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"In":[0],"recent":[1],"years,":[2],"large":[3],"language":[4,20,37],"models":[5],"(LLMs)":[6],"have":[7,31,101],"seen":[8],"rapid":[9],"advancements,":[10],"significantly":[11],"impacting":[12],"various":[13],"fields":[14],"such":[15],"as":[16,69],"computer":[17],"vision,":[18],"natural":[19],"processing,":[21],"and":[22,39,61,72,80,131,147,175,183],"software":[23,47],"engineering.":[24],"These":[25,63],"LLMs,":[26,92],"exemplified":[27],"by":[28,125,162],"OpenAI's":[29],"ChatGPT,":[30],"revolutionized":[32],"the":[33,88,94,116,122,186],"way":[34],"we":[35,114,129,155,169],"approach":[36],"understanding":[38,121],"generation":[40],"tasks.":[41],"However,":[42],"in":[43,58,106],"contrast":[44],"to":[45,76,83,93],"traditional":[46],"development":[48,51],"practices,":[49],"LLM":[50,84,126,163],"introduces":[52],"new":[53],"challenges":[54,64,100,123,160],"for":[55,178],"AI":[56],"developers":[57,75,182],"design,":[59],"implementation,":[60],"deployment.":[62],"span":[65],"different":[66],"areas":[67],"(such":[68],"prompts,":[70],"APIs,":[71],"plugins),":[73],"requiring":[74],"navigate":[77],"unique":[78],"methodologies":[79],"considerations":[81],"specific":[82],"application":[85],"development.":[86],"Despite":[87],"profound":[89],"influence":[90],"of":[91,96,159,173],"best":[95],"our":[97],"knowledge,":[98],"these":[99],"not":[102],"been":[103],"thoroughly":[104],"investigated":[105],"previous":[107],"empirical":[108],"studies.":[109],"To":[110],"fill":[111],"this":[112,167],"gap,":[113],"present":[115],"first":[117,143],"comprehensive":[118],"study":[119],"on":[120,166],"faced":[124,161],"developers.":[127,164],"Specifically,":[128],"crawl":[130],"analyze":[132],"29,057":[133],"relevant":[134],"questions":[135],"from":[136],"a":[137,157,171],"popular":[138],"OpenAI":[139,187],"developer":[140],"forum.":[141],"We":[142],"examine":[144],"their":[145],"popularity":[146],"difficulty.":[148],"After":[149],"manually":[150],"analyzing":[151],"2,364":[152],"sampled":[153],"questions,":[154],"construct":[156],"taxonomy":[158],"Based":[165],"taxonomy,":[168],"summarize":[170],"set":[172],"findings":[174],"actionable":[176],"implications":[177],"LLM-related":[179],"stakeholders,":[180],"including":[181],"providers":[184],"(especially":[185],"organization).":[188]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":10}],"updated_date":"2026-03-30T08:08:38.191290","created_date":"2025-10-10T00:00:00"}