{"id":"https://openalex.org/W4406496877","doi":"https://doi.org/10.1109/bigdata62323.2024.10825139","title":"Predicting ChatGPT\u2019s Ability to Solve Complex Programming Challenges","display_name":"Predicting ChatGPT\u2019s Ability to Solve Complex Programming Challenges","publication_year":2024,"publication_date":"2024-12-15","ids":{"openalex":"https://openalex.org/W4406496877","doi":"https://doi.org/10.1109/bigdata62323.2024.10825139"},"language":"en","primary_location":{"id":"doi:10.1109/bigdata62323.2024.10825139","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata62323.2024.10825139","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Big Data (BigData)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102755635","display_name":"Nguyen Ho","orcid":"https://orcid.org/0000-0002-2308-4329"},"institutions":[{"id":"https://openalex.org/I165556055","display_name":"Loyola University Maryland","ror":"https://ror.org/01by1wp65","country_code":"US","type":"education","lineage":["https://openalex.org/I165556055"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Nguyen Ho","raw_affiliation_strings":["Loyola University Maryland,Department of Computer Science,USA"],"affiliations":[{"raw_affiliation_string":"Loyola University Maryland,Department of Computer Science,USA","institution_ids":["https://openalex.org/I165556055"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115914139","display_name":"James May","orcid":null},"institutions":[{"id":"https://openalex.org/I161171246","display_name":"West Chester University","ror":"https://ror.org/0053n5071","country_code":"US","type":"education","lineage":["https://openalex.org/I161171246"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"James May","raw_affiliation_strings":["West Chester University,Department of Computer Science,USA"],"affiliations":[{"raw_affiliation_string":"West Chester University,Department of Computer Science,USA","institution_ids":["https://openalex.org/I161171246"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115914140","display_name":"Bao Ngo","orcid":null},"institutions":[{"id":"https://openalex.org/I70571728","display_name":"Oberlin College","ror":"https://ror.org/05ac26z88","country_code":"US","type":"education","lineage":["https://openalex.org/I70571728"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bao Ngo","raw_affiliation_strings":["Oberlin College,Department of Computer Science,USA"],"affiliations":[{"raw_affiliation_string":"Oberlin College,Department of Computer Science,USA","institution_ids":["https://openalex.org/I70571728"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115914141","display_name":"Jack Formato","orcid":null},"institutions":[{"id":"https://openalex.org/I165556055","display_name":"Loyola University Maryland","ror":"https://ror.org/01by1wp65","country_code":"US","type":"education","lineage":["https://openalex.org/I165556055"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jack Formato","raw_affiliation_strings":["Loyola University Maryland,Department of Computer Science,USA"],"affiliations":[{"raw_affiliation_string":"Loyola University Maryland,Department of Computer Science,USA","institution_ids":["https://openalex.org/I165556055"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Linh Ngo","orcid":null},"institutions":[{"id":"https://openalex.org/I161171246","display_name":"West Chester University","ror":"https://ror.org/0053n5071","country_code":"US","type":"education","lineage":["https://openalex.org/I161171246"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Linh Ngo","raw_affiliation_strings":["West Chester University,Department of Computer Science,USA"],"affiliations":[{"raw_affiliation_string":"West Chester University,Department of Computer Science,USA","institution_ids":["https://openalex.org/I161171246"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115914142","display_name":"Van Long Ho","orcid":null},"institutions":[{"id":"https://openalex.org/I228151691","display_name":"Ho Chi Minh City International University","ror":"https://ror.org/003szmg30","country_code":"VN","type":"education","lineage":["https://openalex.org/I123565023","https://openalex.org/I228151691"]}],"countries":["VN"],"is_corresponding":false,"raw_author_name":"Van Long Ho","raw_affiliation_strings":["International University,School of Computer Science and Engineering,Vietnam"],"affiliations":[{"raw_affiliation_string":"International University,School of Computer Science and Engineering,Vietnam","institution_ids":["https://openalex.org/I228151691"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5115914143","display_name":"Hoang Bui","orcid":null},"institutions":[{"id":"https://openalex.org/I165556055","display_name":"Loyola University Maryland","ror":"https://ror.org/01by1wp65","country_code":"US","type":"education","lineage":["https://openalex.org/I165556055"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hoang Bui","raw_affiliation_strings":["Loyola University Maryland,Department of Computer Science,USA"],"affiliations":[{"raw_affiliation_string":"Loyola University Maryland,Department of Computer Science,USA","institution_ids":["https://openalex.org/I165556055"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5102755635"],"corresponding_institution_ids":["https://openalex.org/I165556055"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.18988448,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1756","last_page":"1764"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.983299970626831,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.983299970626831,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T13702","display_name":"Machine Learning in Healthcare","score":0.9376000165939331,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9147999882698059,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6674186587333679}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6674186587333679}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/bigdata62323.2024.10825139","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata62323.2024.10825139","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Big Data (BigData)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":48,"referenced_works":["https://openalex.org/W1481128830","https://openalex.org/W1614298861","https://openalex.org/W1949534580","https://openalex.org/W1967129101","https://openalex.org/W1967390364","https://openalex.org/W1987127298","https://openalex.org/W2048587526","https://openalex.org/W2767123881","https://openalex.org/W2990138404","https://openalex.org/W3122283993","https://openalex.org/W3161457214","https://openalex.org/W3177813494","https://openalex.org/W3202712981","https://openalex.org/W4226485558","https://openalex.org/W4231934124","https://openalex.org/W4255528569","https://openalex.org/W4286750487","https://openalex.org/W4287024925","https://openalex.org/W4365205411","https://openalex.org/W4366851162","https://openalex.org/W4367000100","https://openalex.org/W4372271487","https://openalex.org/W4376143487","https://openalex.org/W4385302156","https://openalex.org/W4385902209","https://openalex.org/W4390723974","https://openalex.org/W4392023437","https://openalex.org/W4392484257","https://openalex.org/W4392498330","https://openalex.org/W4392866983","https://openalex.org/W4393212563","https://openalex.org/W6636510571","https://openalex.org/W6641133355","https://openalex.org/W6794686226","https://openalex.org/W6798182279","https://openalex.org/W6800166007","https://openalex.org/W6810874553","https://openalex.org/W6840793632","https://openalex.org/W6851560035","https://openalex.org/W6852318870","https://openalex.org/W6852499657","https://openalex.org/W6852887568","https://openalex.org/W6855386784","https://openalex.org/W6862305990","https://openalex.org/W6862478771","https://openalex.org/W6862593938","https://openalex.org/W6881734288","https://openalex.org/W6980511385"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"The":[0,126],"recent":[1],"emergence":[2],"of":[3,52,64,71,177,193,201,210,229],"Large":[4],"Language":[5],"Model":[6],"(LLM)-based":[7],"tools":[8,41],"such":[9,66,188],"as":[10,67,189],"OpenAI\u2019s":[11],"ChatGPT":[12,120,130,178,224],"and":[13,24,140,151,236],"Google\u2019s":[14],"Gemini":[15],"has":[16],"sparked":[17],"excitement":[18],"across":[19],"the":[20,29,34,50,62,145,156,166,175,190,198,208,213],"software":[21,30,53],"development":[22,31],"industry,":[23],"offered":[25],"promises":[26],"to":[27,48,104,121,134,136,154,231,238],"transform":[28],"process.":[32],"Despite":[33],"enthusiasm,":[35],"it":[36],"remains":[37],"uncertain":[38],"whether":[39],"these":[40,95,123],"are":[42,131],"already":[43],"good":[44],"enough":[45],"at":[46],"coding":[47],"replace":[49],"role":[51],"developers.":[54],"Currently,":[55],"no":[56],"studies":[57],"have":[58],"provided":[59],"insights":[60],"into":[61],"performance":[63,225],"LLMs,":[65],"understanding":[68],"which":[69],"characteristics":[70,158],"a":[72,100,111,194,202],"programming":[73,88,107,113,124,181,195],"task":[74,157],"might":[75],"affect":[76,207],"an":[77,83,227],"LLM's":[78],"performance,":[79],"or":[80,197],"predicting":[81],"how":[82],"LLM":[84],"will":[85],"handle":[86],"new":[87,180],"challenges.":[89],"In":[90],"this":[91],"work,":[92],"we":[93,143,168],"address":[94],"challenges":[96],"by":[97],"first":[98],"creating":[99],"data":[101],"collection":[102],"framework":[103],"gather":[105],"3,323":[106],"tasks":[108],"from":[109,129],"Kattis,":[110],"widely-used":[112],"challenge":[114],"platform.":[115],"We":[116],"then":[117],"use":[118,144],"OpenAI's":[119],"solve":[122],"tasks.":[125],"solutions":[127],"obtained":[128],"submitted":[132],"back":[133],"Kattis":[135],"evaluate":[137],"their":[138],"correctness":[139],"effectiveness.":[141],"Next,":[142],"collected":[146],"data,":[147],"including":[148],"both":[149],"problem":[150,203],"solution":[152],"information,":[153],"analyze":[155],"that":[159,172,186,217],"significantly":[160,206],"influence":[161],"ChatGPT's":[162],"performance.":[163],"Building":[164],"on":[165,179],"analysis,":[167],"develop":[169],"predictive":[170,219],"models":[171],"can":[173,205,221],"forecast":[174],"efficacy":[176,209],"problems.":[182,242],"Our":[183],"analysis":[184],"indicates":[185],"factors":[187],"difficulty":[191],"level":[192],"challenge,":[196],"readability":[199],"complexity":[200],"description":[204],"ChatGPT.":[211],"Finally,":[212],"experimental":[214],"results":[215],"show":[216],"our":[218],"model":[220],"correctly":[222],"predict":[223],"with":[226],"accuracy":[228],"up":[230,237],"90%":[232],"for":[233,240],"easy":[234],"problems,":[235],"79%":[239],"difficult":[241]},"counts_by_year":[],"updated_date":"2026-04-16T08:26:57.006410","created_date":"2025-10-10T00:00:00"}