{"id":"https://openalex.org/W3135120146","doi":"https://doi.org/10.1145/3408877.3432422","title":"Towards a Human-AI Hybrid System for Categorising Programming Problems","display_name":"Towards a Human-AI Hybrid System for Categorising Programming Problems","publication_year":2021,"publication_date":"2021-03-03","ids":{"openalex":"https://openalex.org/W3135120146","doi":"https://doi.org/10.1145/3408877.3432422","mag":"3135120146"},"language":"en","primary_location":{"id":"doi:10.1145/3408877.3432422","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3408877.3432422","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3408877.3432422","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 52nd ACM Technical Symposium on Computer Science Education","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3408877.3432422","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5018752800","display_name":"Filipe Dwan Pereira","orcid":"https://orcid.org/0000-0003-4914-3347"},"institutions":[{"id":"https://openalex.org/I4210144834","display_name":"Universidade Federal de Roraima","ror":"https://ror.org/03ehp1h78","country_code":"BR","type":"education","lineage":["https://openalex.org/I4210144834"]}],"countries":["BR"],"is_corresponding":true,"raw_author_name":"Filipe Dwan Pereira","raw_affiliation_strings":["Federal University of Roraima, Boa Vista, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Federal University of Roraima, Boa Vista, Brazil","institution_ids":["https://openalex.org/I4210144834"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072638126","display_name":"F.M. Andrade Pires","orcid":"https://orcid.org/0000-0002-4802-6360"},"institutions":[{"id":"https://openalex.org/I4210144834","display_name":"Universidade Federal de Roraima","ror":"https://ror.org/03ehp1h78","country_code":"BR","type":"education","lineage":["https://openalex.org/I4210144834"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Francisco Pires","raw_affiliation_strings":["Federal University of Roraima, Boa Vista, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Federal University of Roraima, Boa Vista, Brazil","institution_ids":["https://openalex.org/I4210144834"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000756886","display_name":"Samuel C. Fonseca","orcid":null},"institutions":[{"id":"https://openalex.org/I62885914","display_name":"Universidade Federal do Amazonas","ror":"https://ror.org/02263ky35","country_code":"BR","type":"education","lineage":["https://openalex.org/I62885914"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Samuel C. Fonseca","raw_affiliation_strings":["Federal University of Amazonas, Manaus, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Federal University of Amazonas, Manaus, Brazil","institution_ids":["https://openalex.org/I62885914"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071900519","display_name":"Elaine Oliveira","orcid":"https://orcid.org/0000-0003-2884-9359"},"institutions":[{"id":"https://openalex.org/I62885914","display_name":"Universidade Federal do Amazonas","ror":"https://ror.org/02263ky35","country_code":"BR","type":"education","lineage":["https://openalex.org/I62885914"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Elaine H. T. Oliveira","raw_affiliation_strings":["Federal University of Amazonas, Manaus, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Federal University of Amazonas, Manaus, Brazil","institution_ids":["https://openalex.org/I62885914"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086721948","display_name":"Leandro Silva Galv\u00e3o de Carvalho","orcid":"https://orcid.org/0000-0003-2970-2084"},"institutions":[{"id":"https://openalex.org/I62885914","display_name":"Universidade Federal do Amazonas","ror":"https://ror.org/02263ky35","country_code":"BR","type":"education","lineage":["https://openalex.org/I62885914"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Leandro S. G. Carvalho","raw_affiliation_strings":["Federal University of Amazonas, Manaus, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Federal University of Amazonas, Manaus, Brazil","institution_ids":["https://openalex.org/I62885914"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076321218","display_name":"David Braga Fernandes de Oliveira","orcid":"https://orcid.org/0000-0002-2887-2324"},"institutions":[{"id":"https://openalex.org/I62885914","display_name":"Universidade Federal do Amazonas","ror":"https://ror.org/02263ky35","country_code":"BR","type":"education","lineage":["https://openalex.org/I62885914"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"David B. F. Oliveira","raw_affiliation_strings":["Federal University of Amazonas, Manaus, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Federal University of Amazonas, Manaus, Brazil","institution_ids":["https://openalex.org/I62885914"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5034125179","display_name":"Alexandra I. Cristea","orcid":"https://orcid.org/0000-0002-1454-8822"},"institutions":[{"id":"https://openalex.org/I190082696","display_name":"Durham University","ror":"https://ror.org/01v29qb04","country_code":"GB","type":"education","lineage":["https://openalex.org/I190082696"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Alexandra I. Cristea","raw_affiliation_strings":["Durham University, Durham, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Durham University, Durham, United Kingdom","institution_ids":["https://openalex.org/I190082696"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5018752800"],"corresponding_institution_ids":["https://openalex.org/I4210144834"],"apc_list":null,"apc_paid":null,"fwci":4.8348,"has_fulltext":true,"cited_by_count":20,"citation_normalized_percentile":{"value":0.9516721,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"94","last_page":"100"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11652","display_name":"Imbalanced Data Classification Techniques","score":0.9882000088691711,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.9811999797821045,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7457401156425476},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5765663981437683},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.5189335346221924},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.5025110244750977},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.46867188811302185},{"id":"https://openalex.org/keywords/online-learning","display_name":"Online learning","score":0.4604834020137787},{"id":"https://openalex.org/keywords/complement","display_name":"Complement (music)","score":0.4428473114967346},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.413411021232605},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.2502875328063965},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.19144192337989807},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.08684957027435303}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7457401156425476},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5765663981437683},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.5189335346221924},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.5025110244750977},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.46867188811302185},{"id":"https://openalex.org/C2986087404","wikidata":"https://www.wikidata.org/wiki/Q15946010","display_name":"Online learning","level":2,"score":0.4604834020137787},{"id":"https://openalex.org/C112313634","wikidata":"https://www.wikidata.org/wiki/Q7886648","display_name":"Complement (music)","level":5,"score":0.4428473114967346},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.413411021232605},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.2502875328063965},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.19144192337989807},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.08684957027435303},{"id":"https://openalex.org/C188082640","wikidata":"https://www.wikidata.org/wiki/Q1780899","display_name":"Complementation","level":4,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C127716648","wikidata":"https://www.wikidata.org/wiki/Q104053","display_name":"Phenotype","level":3,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3408877.3432422","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3408877.3432422","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3408877.3432422","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 52nd ACM Technical Symposium on Computer Science Education","raw_type":"proceedings-article"},{"id":"pmh:oai:dro.dur.ac.uk.OAI2:34413","is_oa":false,"landing_page_url":"http://dro.dur.ac.uk/34413/","pdf_url":null,"source":{"id":"https://openalex.org/S4377196258","display_name":"Durham Research Online (Durham University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I190082696","host_organization_name":"Durham University","host_organization_lineage":["https://openalex.org/I190082696"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":" SIGCSE '21: Proceedings of the 52nd ACM Technical Symposium on Computer Science Education. : ACM, pp. 94-100","raw_type":"Book chapter"},{"id":"pmh:oai:durham-repository.worktribe.com:1140204","is_oa":true,"landing_page_url":"https://durham-repository.worktribe.com/output/1140204","pdf_url":null,"source":{"id":"https://openalex.org/S4306400188","display_name":"Durham Research Online (Durham University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I190082696","host_organization_name":"Durham University","host_organization_lineage":["https://openalex.org/I190082696"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"publishedVersion"}],"best_oa_location":{"id":"doi:10.1145/3408877.3432422","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3408877.3432422","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3408877.3432422","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 52nd ACM Technical Symposium on Computer Science Education","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.7900000214576721,"display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G3156316819","display_name":null,"funder_award_id":"001/2020","funder_id":"https://openalex.org/F4320321091","funder_display_name":"Coordena\u00e7\u00e3o de Aperfei\u00e7oamento de Pessoal de N\u00edvel Superior"},{"id":"https://openalex.org/G3559415485","display_name":null,"funder_award_id":"003/2019","funder_id":"https://openalex.org/F4320332195","funder_display_name":"Samsung"},{"id":"https://openalex.org/G4257313756","display_name":null,"funder_award_id":"8.387/1991","funder_id":"https://openalex.org/F4320332195","funder_display_name":"Samsung"},{"id":"https://openalex.org/G4588099766","display_name":null,"funder_award_id":"001/2020","funder_id":"https://openalex.org/F4320332195","funder_display_name":"Samsung"},{"id":"https://openalex.org/G6087244377","display_name":null,"funder_award_id":"001/2020","funder_id":"https://openalex.org/F4320319527","funder_display_name":"Samsung Eletr\u00f4nica da Amaz\u00f4nia"}],"funders":[{"id":"https://openalex.org/F4320318869","display_name":"Universidade Federal do Amazonas","ror":"https://ror.org/02263ky35"},{"id":"https://openalex.org/F4320319527","display_name":"Samsung Eletr\u00f4nica da Amaz\u00f4nia","ror":null},{"id":"https://openalex.org/F4320321091","display_name":"Coordena\u00e7\u00e3o de Aperfei\u00e7oamento de Pessoal de N\u00edvel Superior","ror":"https://ror.org/00x0ma614"},{"id":"https://openalex.org/F4320332195","display_name":"Samsung","ror":"https://ror.org/04w3jy968"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3135120146.pdf","grobid_xml":"https://content.openalex.org/works/W3135120146.grobid-xml"},"referenced_works_count":39,"referenced_works":["https://openalex.org/W80323479","https://openalex.org/W226296531","https://openalex.org/W1553976193","https://openalex.org/W2083442042","https://openalex.org/W2100075873","https://openalex.org/W2153579005","https://openalex.org/W2182887552","https://openalex.org/W2250539671","https://openalex.org/W2562319768","https://openalex.org/W2581574654","https://openalex.org/W2605360097","https://openalex.org/W2677564260","https://openalex.org/W2736274677","https://openalex.org/W2767106145","https://openalex.org/W2767123881","https://openalex.org/W2792805964","https://openalex.org/W2808220452","https://openalex.org/W2884745705","https://openalex.org/W2901067297","https://openalex.org/W2911295267","https://openalex.org/W2947495225","https://openalex.org/W2947911750","https://openalex.org/W2950250582","https://openalex.org/W2954138110","https://openalex.org/W2965237463","https://openalex.org/W2965905350","https://openalex.org/W2972049724","https://openalex.org/W2987220306","https://openalex.org/W2988747964","https://openalex.org/W2991272671","https://openalex.org/W2992113183","https://openalex.org/W2996497876","https://openalex.org/W3003537538","https://openalex.org/W3028243744","https://openalex.org/W3033715984","https://openalex.org/W3122283993","https://openalex.org/W3188237241","https://openalex.org/W4234170634","https://openalex.org/W4301154865"],"related_works":["https://openalex.org/W2382566571","https://openalex.org/W4390516098","https://openalex.org/W2349321798","https://openalex.org/W2366686860","https://openalex.org/W2181948922","https://openalex.org/W2350859087","https://openalex.org/W2387118502","https://openalex.org/W2384362569","https://openalex.org/W4233775131","https://openalex.org/W3037187668"],"abstract_inverted_index":{"As":[0,165],"programming":[1,36,70],"skills":[2],"are":[3,27,65,89],"increasingly":[4],"required":[5],"world-wide":[6],"and":[7,81,95,129],"across":[8],"disciplines,":[9],"many":[10,40],"students":[11,42,78],"use":[12,193],"online":[13],"platforms":[14],"that":[15,60,88],"provide":[16],"automatic":[17],"feedback":[18],"through":[19],"a":[20,103,157,166,184],"Programming":[21],"Online":[22],"Judge":[23],"(POJ)":[24],"mechanism.":[25],"POJs":[26],"very":[28],"popular":[29],"e-learning":[30],"tools,":[31],"boasting":[32],"large":[33],"collections":[34],"of":[35,57,63,111,149,168],"problems.":[37,201],"Despite":[38],"their":[39,51,93],"benefits,":[41],"often":[43],"struggle":[44],"when":[45],"solving":[46],"problems":[47,64],"not":[48,66,90],"compatible":[49],"with":[50,146,162],"prior":[52],"knowledge.":[53],"One":[54],"important":[55],"cause":[56],"this":[58],"is":[59],"usually":[61],"statements":[62,128],"classified":[67],"according":[68],"to":[69,85,92,98,108,174,196],"topics":[71,110],"(paradigms,":[72],"data":[73],"structures,":[74],"etc.)":[75],"and,":[76],"hence,":[77],"waste":[79],"time":[80],"effort":[82],"in":[83,156],"trying":[84],"solve":[86],"exercises":[87],"tailored":[91],"level":[94],"needs.":[96],"Thus,":[97],"support":[99],"students,":[100],"we":[101,170],"propose":[102],"new,":[104],"\"front-heavy\"":[105],"pipeline":[106],"method":[107],"predict":[109],"POJ":[112],"problems,":[113],"using":[114,151],"Bidirectional":[115],"Encoder":[116],"Representations":[117],"from":[118],"Transformers":[119],"(BERT)":[120],"for":[121,132,137,188],"contextual":[122],"text":[123],"augmentation":[124],"over":[125],"the":[126,198],"problem":[127,161],"further":[130],"allowing":[131],"(lighter-weight)":[133],"classical":[134],"machine":[135],"learning":[136],"classification.":[138],"Our":[139],"model":[140,179],"outperformed":[141],"all":[142],"current":[143],"state-of-the":[144],"art,":[145],"an":[147,172],"F1-score":[148],"86%":[150],"stratified":[152],"10":[153],"fold":[154],"cross-validation":[155],"classically":[158],"challenging":[159],"multi-classification":[160],"seven":[163],"categories.":[164],"proof":[167],"concept,":[169],"conducted":[171],"experiment":[173],"show":[175],"how":[176],"our":[177],"predictive":[178],"can":[180],"be":[181],"used":[182],"as":[183],"human-AI":[185],"hybrid":[186],"complement":[187],"POJ,":[189],"where":[190],"learners":[191],"would":[192],"AI-based":[194],"recommendations":[195],"find":[197],"most":[199],"appropriate":[200]},"counts_by_year":[{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":6}],"updated_date":"2026-05-10T08:33:47.465468","created_date":"2025-10-10T00:00:00"}