{"id":"https://openalex.org/W4403413321","doi":"https://doi.org/10.1145/3674805.3686689","title":"Are Large Language Models a Threat to Programming Platforms? An Exploratory Study","display_name":"Are Large Language Models a Threat to Programming Platforms? An Exploratory Study","publication_year":2024,"publication_date":"2024-10-15","ids":{"openalex":"https://openalex.org/W4403413321","doi":"https://doi.org/10.1145/3674805.3686689"},"language":"en","primary_location":{"id":"doi:10.1145/3674805.3686689","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3674805.3686689","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 18th ACM/IEEE International Symposium on Empirical Software Engineering and Measurement","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2409.05824","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5114271563","display_name":"Md Mustakim Billah","orcid":"https://orcid.org/0009-0007-0654-151X"},"institutions":[{"id":"https://openalex.org/I32625721","display_name":"University of Saskatchewan","ror":"https://ror.org/010x8gc63","country_code":"CA","type":"education","lineage":["https://openalex.org/I32625721"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Md Mustakim Billah","raw_affiliation_strings":["University of Saskatchewan, Canada"],"raw_orcid":"https://orcid.org/0009-0007-0654-151X","affiliations":[{"raw_affiliation_string":"University of Saskatchewan, Canada","institution_ids":["https://openalex.org/I32625721"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058812722","display_name":"Palash Ranjan Roy","orcid":"https://orcid.org/0000-0001-9470-4233"},"institutions":[{"id":"https://openalex.org/I32625721","display_name":"University of Saskatchewan","ror":"https://ror.org/010x8gc63","country_code":"CA","type":"education","lineage":["https://openalex.org/I32625721"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Palash Ranjan Roy","raw_affiliation_strings":["University of Saskatchewan, Canada"],"raw_orcid":"https://orcid.org/0000-0001-9470-4233","affiliations":[{"raw_affiliation_string":"University of Saskatchewan, Canada","institution_ids":["https://openalex.org/I32625721"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022264168","display_name":"Zadia Codabux","orcid":"https://orcid.org/0000-0001-6715-3341"},"institutions":[{"id":"https://openalex.org/I32625721","display_name":"University of Saskatchewan","ror":"https://ror.org/010x8gc63","country_code":"CA","type":"education","lineage":["https://openalex.org/I32625721"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Zadia Codabux","raw_affiliation_strings":["University of Saskatchewan, Canada"],"raw_orcid":"https://orcid.org/0000-0001-6715-3341","affiliations":[{"raw_affiliation_string":"University of Saskatchewan, Canada","institution_ids":["https://openalex.org/I32625721"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5015470184","display_name":"Banani Roy","orcid":"https://orcid.org/0000-0003-1247-7781"},"institutions":[{"id":"https://openalex.org/I32625721","display_name":"University of Saskatchewan","ror":"https://ror.org/010x8gc63","country_code":"CA","type":"education","lineage":["https://openalex.org/I32625721"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Banani Roy","raw_affiliation_strings":["University of Saskatchewan, Canada"],"raw_orcid":"https://orcid.org/0000-0003-1247-7781","affiliations":[{"raw_affiliation_string":"University of Saskatchewan, Canada","institution_ids":["https://openalex.org/I32625721"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I32625721"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"292","last_page":"301"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.9937999844551086,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.9872000217437744,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7577903270721436},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.4538617432117462},{"id":"https://openalex.org/keywords/exploratory-research","display_name":"Exploratory research","score":0.42411312460899353}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7577903270721436},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.4538617432117462},{"id":"https://openalex.org/C85973986","wikidata":"https://www.wikidata.org/wiki/Q1091731","display_name":"Exploratory research","level":2,"score":0.42411312460899353},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C19165224","wikidata":"https://www.wikidata.org/wiki/Q23404","display_name":"Anthropology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3674805.3686689","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3674805.3686689","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 18th ACM/IEEE International Symposium on Empirical Software Engineering and Measurement","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2409.05824","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2409.05824","pdf_url":"https://arxiv.org/pdf/2409.05824","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2409.05824","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2409.05824","pdf_url":"https://arxiv.org/pdf/2409.05824","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320334593","display_name":"Natural Sciences and Engineering Research Council of Canada","ror":"https://ror.org/01h531d29"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4403413321.pdf","grobid_xml":"https://content.openalex.org/works/W4403413321.grobid-xml"},"referenced_works_count":21,"referenced_works":["https://openalex.org/W2767123881","https://openalex.org/W3011584400","https://openalex.org/W3122283993","https://openalex.org/W3166295544","https://openalex.org/W3166421076","https://openalex.org/W4211263275","https://openalex.org/W4221143046","https://openalex.org/W4225108562","https://openalex.org/W4281392476","https://openalex.org/W4283321056","https://openalex.org/W4312438588","https://openalex.org/W4312900476","https://openalex.org/W4366316542","https://openalex.org/W4383196665","https://openalex.org/W4389544307","https://openalex.org/W4391386032","https://openalex.org/W4393034946","https://openalex.org/W4395029328","https://openalex.org/W6811216112","https://openalex.org/W6851275496","https://openalex.org/W6903968576"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Background:":[0],"Competitive":[1],"programming":[2,14,55,68,71,100,268,287],"platforms":[3,21,72,269,288],"such":[4],"as":[5,22],"LeetCode,":[6],"Codeforces,":[7,113],"and":[8,41,84,92,110,117,130,156,224,239],"HackerRank":[9,139,197],"provide":[10],"challenges":[11,69,191],"to":[12,48,61,65,89,96,140,150,165,255,290],"evaluate":[13],"skills.":[15],"Technical":[16],"recruiters":[17],"frequently":[18],"utilize":[19],"these":[20],"a":[23,46,261,272],"criterion":[24],"for":[25,274],"screening":[26],"resumes.":[27],"With":[28,276],"the":[29,54,171,263,277,285],"recent":[30],"advent":[31],"of":[32,265,279],"advanced":[33],"Large":[34],"Language":[35],"Models":[36],"(LLMs)":[37],"like":[38],"ChatGPT,":[39],"Gemini,":[40],"Meta":[42],"AI,":[43],"there":[44],"is":[45,270],"need":[47,289],"assess":[49,62],"their":[50,80],"problem-solving":[51],"ability":[52],"on":[53,138,183,192,205,267],"platforms.":[56,101],"Aims:":[57],"This":[58,103],"study":[59,104],"aims":[60],"LLMs\u2019":[63,144],"capability":[64],"solve":[66],"diverse":[67,208],"across":[70,215],"with":[73,221],"varying":[74,118],"difficulty":[75,119],"levels,":[76],"providing":[77],"insights":[78,142],"into":[79,143],"performance":[81,209,245,264],"in":[82,99,124,153,173,196,201,236,246,250,284],"real-time":[83,145],"offline":[85],"scenarios,":[86],"comparing":[87],"them":[88],"human":[90],"programmers,":[91],"identifying":[93],"potential":[94],"threats":[95],"established":[97],"norms":[98],"Method:":[102],"utilized":[105],"98":[106],"problems":[107,223],"from":[108,112,128],"LeetCode":[109,184,229],"126":[111],"covering":[114],"15":[115],"categories":[116],"levels.":[120],"Then,":[121],"we":[122],"participated":[123],"nine":[125],"online":[126],"contests":[127,253],"Codeforces":[129,252],"LeetCode.":[131],"Finally,":[132],"two":[133],"certification":[134],"tests":[135],"were":[136,148,160],"attempted":[137],"gain":[141],"performance.":[146],"Prompts":[147],"used":[149],"guide":[151],"LLMs":[152,172,177,219,232,266],"solving":[154],"problems,":[155,231],"iterative":[157],"feedback":[158],"mechanisms":[159],"employed.":[161],"We":[162],"also":[163],"tried":[164],"find":[166],"any":[167],"possible":[168],"correlation":[169],"among":[170],"different":[174],"scenarios.":[175],"Results:":[176],"generally":[178,233],"achieved":[179],"higher":[180],"success":[181],"rates":[182],"(e.g.,":[185],"ChatGPT":[186,211],"at":[187],"71.43%)":[188],"but":[189,242],"faced":[190],"Codeforces.":[193,206],"While":[194,258],"excelling":[195],"certifications,":[198],"they":[199],"struggled":[200,220],"virtual":[202],"contests,":[203,248],"especially":[204],"Despite":[207],"trends,":[210],"consistently":[212],"performed":[213],"well":[214],"categories,":[216],"yet":[217],"all":[218],"harder":[222,251],"lower":[225],"acceptance":[226],"rates.":[227],"In":[228],"archive":[230],"outperformed":[234],"users":[235],"time":[237],"efficiency":[238],"memory":[240],"usage":[241],"exhibited":[243],"moderate":[244],"live":[247],"particularly":[249],"compared":[254],"humans.":[256],"Conclusions:":[257],"not":[259],"necessarily":[260],"threat,":[262],"indeed":[271],"cause":[273],"concern.":[275],"prospect":[278],"more":[280],"efficient":[281],"models":[282],"emerging":[283],"future,":[286],"address":[291],"this":[292],"issue":[293],"promptly.":[294]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2024-10-16T00:00:00"}