{"id":"https://openalex.org/W2792805964","doi":"https://doi.org/10.1145/3158670","title":"Automatically Learning Topics and Difficulty Levels of Problems in Online Judge Systems","display_name":"Automatically Learning Topics and Difficulty Levels of Problems in Online Judge Systems","publication_year":2018,"publication_date":"2018-03-07","ids":{"openalex":"https://openalex.org/W2792805964","doi":"https://doi.org/10.1145/3158670","mag":"2792805964"},"language":"en","primary_location":{"id":"doi:10.1145/3158670","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3158670","pdf_url":null,"source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://publications.aston.ac.uk/id/eprint/31880/11/finalVersion.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5037145565","display_name":"Wayne Xin Zhao","orcid":"https://orcid.org/0000-0002-8333-6196"},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wayne Xin Zhao","raw_affiliation_strings":["Renmin University of China, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Renmin University of China, Beijing, China","institution_ids":["https://openalex.org/I78988378"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100399717","display_name":"Wenhui Zhang","orcid":"https://orcid.org/0000-0002-3768-1917"},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenhui Zhang","raw_affiliation_strings":["Renmin University of China, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Renmin University of China, Beijing, China","institution_ids":["https://openalex.org/I78988378"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015709853","display_name":"Yulan He","orcid":"https://orcid.org/0000-0003-3948-5845"},"institutions":[{"id":"https://openalex.org/I169199633","display_name":"Aston University","ror":"https://ror.org/05j0ve876","country_code":"GB","type":"education","lineage":["https://openalex.org/I169199633"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Yulan He","raw_affiliation_strings":["Aston University, Birmingham, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Aston University, Birmingham, United Kingdom","institution_ids":["https://openalex.org/I169199633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044651577","display_name":"Xing Xie","orcid":"https://orcid.org/0000-0002-8608-8482"},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xing Xie","raw_affiliation_strings":["Microsoft Research, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research, Beijing, China","institution_ids":["https://openalex.org/I4210113369"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5025631695","display_name":"Ji-Rong Wen","orcid":"https://orcid.org/0000-0002-9777-9676"},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ji-Rong Wen","raw_affiliation_strings":["Renmin University of China, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Renmin University of China, Beijing, China","institution_ids":["https://openalex.org/I78988378"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":5.2261,"has_fulltext":true,"cited_by_count":67,"citation_normalized_percentile":{"value":0.96156308,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"36","issue":"3","first_page":"1","last_page":"33"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9944999814033508,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7451010942459106},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5264283418655396},{"id":"https://openalex.org/keywords/online-learning","display_name":"Online learning","score":0.4831628203392029},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.4492664039134979},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.38491058349609375},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.33895695209503174},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.23758423328399658}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7451010942459106},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5264283418655396},{"id":"https://openalex.org/C2986087404","wikidata":"https://www.wikidata.org/wiki/Q15946010","display_name":"Online learning","level":2,"score":0.4831628203392029},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4492664039134979},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.38491058349609375},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.33895695209503174},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.23758423328399658},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3158670","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3158670","pdf_url":null,"source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},{"id":"pmh:oai:wrap.warwick.ac.uk:108978","is_oa":false,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4306400665","display_name":"Warwick Research Archive Portal (University of Warwick)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I39555362","host_organization_name":"University of Warwick","host_organization_lineage":["https://openalex.org/I39555362"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"","raw_type":"Journal Article"},{"id":"pmh:oai:publications.aston.ac.uk:31880","is_oa":true,"landing_page_url":"https://publications.aston.ac.uk/view/author/0eaf71d33f36ad02379e16440bd5bcb2.html>,","pdf_url":"https://publications.aston.ac.uk/id/eprint/31880/11/finalVersion.pdf","source":{"id":"https://openalex.org/S4306400483","display_name":"Aston Publications Explorer (Aston University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I169199633","host_organization_name":"Aston University","host_organization_lineage":["https://openalex.org/I169199633"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"}],"best_oa_location":{"id":"pmh:oai:publications.aston.ac.uk:31880","is_oa":true,"landing_page_url":"https://publications.aston.ac.uk/view/author/0eaf71d33f36ad02379e16440bd5bcb2.html>,","pdf_url":"https://publications.aston.ac.uk/id/eprint/31880/11/finalVersion.pdf","source":{"id":"https://openalex.org/S4306400483","display_name":"Aston Publications Explorer (Aston University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I169199633","host_organization_name":"Aston University","host_organization_lineage":["https://openalex.org/I169199633"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.7900000214576721}],"awards":[{"id":"https://openalex.org/G4975599656","display_name":null,"funder_award_id":"2014CB340403","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"},{"id":"https://openalex.org/G5859226697","display_name":null,"funder_award_id":"61502502","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320308943","display_name":"Microsoft Research","ror":"https://ror.org/00d0nc645"},{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2792805964.pdf","grobid_xml":"https://content.openalex.org/works/W2792805964.grobid-xml"},"referenced_works_count":68,"referenced_works":["https://openalex.org/W11388158","https://openalex.org/W62042437","https://openalex.org/W64872011","https://openalex.org/W74742851","https://openalex.org/W125847312","https://openalex.org/W226296531","https://openalex.org/W568024311","https://openalex.org/W1498269992","https://openalex.org/W1597703949","https://openalex.org/W1880262756","https://openalex.org/W1883949185","https://openalex.org/W1911642898","https://openalex.org/W1959691478","https://openalex.org/W1973196098","https://openalex.org/W1985824294","https://openalex.org/W1985854669","https://openalex.org/W2001082470","https://openalex.org/W2008064904","https://openalex.org/W2008733342","https://openalex.org/W2009779426","https://openalex.org/W2015040676","https://openalex.org/W2022149269","https://openalex.org/W2023164354","https://openalex.org/W2027711578","https://openalex.org/W2039552226","https://openalex.org/W2039625536","https://openalex.org/W2044653300","https://openalex.org/W2051963836","https://openalex.org/W2061802017","https://openalex.org/W2061873838","https://openalex.org/W2099013648","https://openalex.org/W2099213975","https://openalex.org/W2100075873","https://openalex.org/W2100163972","https://openalex.org/W2107743791","https://openalex.org/W2111760158","https://openalex.org/W2112971401","https://openalex.org/W2117122404","https://openalex.org/W2132255457","https://openalex.org/W2132726007","https://openalex.org/W2136208491","https://openalex.org/W2137245235","https://openalex.org/W2140124448","https://openalex.org/W2140310134","https://openalex.org/W2149427297","https://openalex.org/W2153975459","https://openalex.org/W2160920329","https://openalex.org/W2162646781","https://openalex.org/W2163644476","https://openalex.org/W2164545125","https://openalex.org/W2171279286","https://openalex.org/W2259031064","https://openalex.org/W2340855322","https://openalex.org/W2380648751","https://openalex.org/W2404715899","https://openalex.org/W2574744083","https://openalex.org/W2579758603","https://openalex.org/W2605350416","https://openalex.org/W2605961645","https://openalex.org/W2608239929","https://openalex.org/W2727426218","https://openalex.org/W3023847089","https://openalex.org/W3099640513","https://openalex.org/W4231510805","https://openalex.org/W4233135949","https://openalex.org/W4245044266","https://openalex.org/W4301934563","https://openalex.org/W4392399704"],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W4387369504","https://openalex.org/W3046775127","https://openalex.org/W4394896187","https://openalex.org/W3170094116","https://openalex.org/W4386462264","https://openalex.org/W3107602296","https://openalex.org/W4364306694","https://openalex.org/W4312192474"],"abstract_inverted_index":{"Online":[0,28],"Judge":[1],"(OJ)":[2],"systems":[3,32],"have":[4,195],"been":[5],"widely":[6],"used":[7],"in":[8,44,52,70,76,104,137,202],"many":[9],"areas,":[10],"including":[11,206],"programming,":[12],"mathematical":[13],"problems":[14,48,69,103,119,160],"solving,":[15],"and":[16,54,213],"job":[17],"interviews.":[18],"Unlike":[19],"other":[20],"online":[21,159,175],"learning":[22,37,87,96,166],"systems,":[23,47],"such":[24],"as":[25],"Massive":[26],"Open":[27],"Course,":[29],"most":[30,45],"OJ":[31,46,193],"are":[33,49,74,93],"designed":[34],"for":[35],"self-directed":[36],"without":[38],"the":[39,71,109,121,135,156,164,171,184,197],"intervention":[40],"of":[41,60,78,113,158,174,199],"teachers.":[42],"Also,":[43],"simply":[50],"listed":[51],"volumes":[53],"there":[55,92],"is":[56,132],"no":[57],"clear":[58],"organization":[59],"them":[61],"by":[62,161],"topics":[63,79,115,157],"or":[64,80,116],"difficulty":[65,81,172],"levels.":[66,82],"As":[67],"such,":[68],"same":[72,110,122],"volume":[73,111],"mixed":[75],"terms":[77],"By":[83],"analyzing":[84],"large-scale":[85],"users\u2019":[86],"traces,":[88],"we":[89,145,177],"observe":[90],"that":[91],"two":[94,165],"major":[95],"modes":[97],"(or":[98],"patterns).":[99],"Users":[100],"either":[101],"practice":[102],"a":[105,147,179],"sequential":[106],"manner":[107],"from":[108],"regardless":[112],"their":[114],"they":[117],"attempt":[118],"about":[120],"topic,":[123],"which":[124],"may":[125],"spread":[126],"across":[127],"multiple":[128],"volumes.":[129],"Our":[130],"observation":[131],"consistent":[133],"with":[134],"findings":[136],"classic":[138],"educational":[139],"psychology.":[140],"Based":[141],"on":[142,190],"our":[143,200],"observation,":[144],"propose":[146,178],"novel":[148],"two-mode":[149],"Markov":[150],"topic":[151,186,208],"model":[152,182],"to":[153],"automatically":[154],"detect":[155],"jointly":[162],"characterizing":[163],"modes.":[167],"For":[168],"further":[169],"predicting":[170],"level":[173],"problems,":[176],"competition-based":[180],"expertise":[181,210],"using":[183],"learned":[185],"information.":[187],"Extensive":[188],"experiments":[189],"three":[191,203],"large":[192],"datasets":[194],"demonstrated":[196],"effectiveness":[198],"approach":[201],"different":[204],"tasks,":[205],"skill":[207],"extraction,":[209],"competition":[211],"prediction":[212],"problem":[214],"recommendation.":[215]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":7},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":14},{"year":2022,"cited_by_count":10},{"year":2021,"cited_by_count":11},{"year":2020,"cited_by_count":7},{"year":2019,"cited_by_count":8},{"year":2018,"cited_by_count":2}],"updated_date":"2026-07-23T08:03:31.855105","created_date":"2025-10-10T00:00:00"}