{"id":"https://openalex.org/W3022044005","doi":"https://doi.org/10.1109/access.2020.2990980","title":"Building a Comprehensive Automated Programming Assessment System","display_name":"Building a Comprehensive Automated Programming Assessment System","publication_year":2020,"publication_date":"2020-01-01","ids":{"openalex":"https://openalex.org/W3022044005","doi":"https://doi.org/10.1109/access.2020.2990980","mag":"3022044005"},"language":"en","primary_location":{"id":"doi:10.1109/access.2020.2990980","doi":"https://doi.org/10.1109/access.2020.2990980","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2020.2990980","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/8948470/09079865.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-scientific","cwts-core","doaj","jufo-1","ki-jl-1","norway-1","oarl","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"volume":null,"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://ieeexplore.ieee.org/ielx7/6287639/8948470/09079865.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5067736091","display_name":"Igor Mekterovi\u0107","orcid":"https://orcid.org/0000-0002-9813-1935","observed_orcids":["https://orcid.org/0000-0002-9813-1935"]},"institutions":[{"id":"https://openalex.org/I181343428","display_name":"University of Zagreb","ror":"https://ror.org/00mv6sv71","country_code":"HR","type":"education","lineage":["https://openalex.org/I181343428"]},{"id":"https://openalex.org/I4390039327","display_name":"Faculty of Electrical Engineering and Computing in Zagreb","ror":"https://ror.org/04e90ca47","country_code":"HR","type":"education","lineage":["https://openalex.org/I181343428","https://openalex.org/I4390039327"]}],"countries":["HR"],"is_corresponding":false,"raw_author_name":"Igor Mekterovic","raw_affiliation_strings":["Faculty of Electrical Engineering and Computing, University of Zagreb, Zagreb, Croatia"],"raw_orcid":"https://orcid.org/0000-0002-9813-1935","affiliations":[{"raw_affiliation_string":"Faculty of Electrical Engineering and Computing, University of Zagreb, Zagreb, Croatia","institution_ids":["https://openalex.org/I181343428","https://openalex.org/I4390039327"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051434230","display_name":"Ljiljana Brki\u0107","orcid":"https://orcid.org/0000-0002-6345-8735","observed_orcids":["https://orcid.org/0000-0002-6345-8735"]},"institutions":[{"id":"https://openalex.org/I181343428","display_name":"University of Zagreb","ror":"https://ror.org/00mv6sv71","country_code":"HR","type":"education","lineage":["https://openalex.org/I181343428"]},{"id":"https://openalex.org/I4390039327","display_name":"Faculty of Electrical Engineering and Computing in Zagreb","ror":"https://ror.org/04e90ca47","country_code":"HR","type":"education","lineage":["https://openalex.org/I181343428","https://openalex.org/I4390039327"]}],"countries":["HR"],"is_corresponding":false,"raw_author_name":"Ljiljana Brkic","raw_affiliation_strings":["Faculty of Electrical Engineering and Computing, University of Zagreb, Zagreb, Croatia"],"raw_orcid":"https://orcid.org/0000-0002-6345-8735","affiliations":[{"raw_affiliation_string":"Faculty of Electrical Engineering and Computing, University of Zagreb, Zagreb, Croatia","institution_ids":["https://openalex.org/I181343428","https://openalex.org/I4390039327"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085883416","display_name":"Boris Mila\u0161inovi\u0107","orcid":"https://orcid.org/0000-0002-7889-3131","observed_orcids":["https://orcid.org/0000-0002-7889-3131"]},"institutions":[{"id":"https://openalex.org/I181343428","display_name":"University of Zagreb","ror":"https://ror.org/00mv6sv71","country_code":"HR","type":"education","lineage":["https://openalex.org/I181343428"]},{"id":"https://openalex.org/I4390039327","display_name":"Faculty of Electrical Engineering and Computing in Zagreb","ror":"https://ror.org/04e90ca47","country_code":"HR","type":"education","lineage":["https://openalex.org/I181343428","https://openalex.org/I4390039327"]}],"countries":["HR"],"is_corresponding":false,"raw_author_name":"Boris Milasinovic","raw_affiliation_strings":["Faculty of Electrical Engineering and Computing, University of Zagreb, Zagreb, Croatia"],"raw_orcid":"https://orcid.org/0000-0002-7889-3131","affiliations":[{"raw_affiliation_string":"Faculty of Electrical Engineering and Computing, University of Zagreb, Zagreb, Croatia","institution_ids":["https://openalex.org/I181343428","https://openalex.org/I4390039327"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5056642319","display_name":"Mirta Baranovi\u0107","orcid":"https://orcid.org/0000-0002-8941-5485","observed_orcids":["https://orcid.org/0000-0002-8941-5485"]},"institutions":[{"id":"https://openalex.org/I181343428","display_name":"University of Zagreb","ror":"https://ror.org/00mv6sv71","country_code":"HR","type":"education","lineage":["https://openalex.org/I181343428"]},{"id":"https://openalex.org/I4390039327","display_name":"Faculty of Electrical Engineering and Computing in Zagreb","ror":"https://ror.org/04e90ca47","country_code":"HR","type":"education","lineage":["https://openalex.org/I181343428","https://openalex.org/I4390039327"]}],"countries":["HR"],"is_corresponding":false,"raw_author_name":"Mirta Baranovic","raw_affiliation_strings":["Faculty of Electrical Engineering and Computing, University of Zagreb, Zagreb, Croatia"],"raw_orcid":"https://orcid.org/0000-0002-8941-5485","affiliations":[{"raw_affiliation_string":"Faculty of Electrical Engineering and Computing, University of Zagreb, Zagreb, Croatia","institution_ids":["https://openalex.org/I181343428","https://openalex.org/I4390039327"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":1750,"currency":"USD","value_usd":1750},"apc_paid":{"value":1750,"currency":"USD","value_usd":1750},"fwci":9.3513,"has_fulltext":true,"cited_by_count":74,"citation_normalized_percentile":{"value":0.97816425,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":100},"biblio":{"volume":"8","issue":null,"first_page":"81154","last_page":"81172"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9878000020980835,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9878000020980835,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14025","display_name":"Educational Technology and Assessment","score":0.009200000204145908,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.000699999975040555,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automated-programming-assessment","display_name":"automated programming assessment","score":0.9869999885559082},{"id":"https://openalex.org/keywords/edgar","display_name":"EDGAR","score":0.7739999890327454},{"id":"https://openalex.org/keywords/open-source-software","display_name":"open source software","score":0.8930000066757202},{"id":"https://openalex.org/keywords/programming-exercises","display_name":"programming exercises","score":0.734000027179718},{"id":"https://openalex.org/keywords/automated-grading","display_name":"automated grading","score":0.550000011920929},{"id":"https://openalex.org/keywords/academic-integrity","display_name":"academic integrity","score":0.652999997138977},{"id":"https://openalex.org/keywords/multiple-choice-questions","display_name":"multiple-choice questions","score":0.671999990940094},{"id":"https://openalex.org/keywords/programming-education","display_name":"programming education","score":0.49900001287460327},{"id":"https://openalex.org/keywords/software-architecture","display_name":"software architecture","score":0.7170000076293945}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8462041616439819},{"id":"https://openalex.org/C548217200","wikidata":"https://www.wikidata.org/wiki/Q251","display_name":"Java","level":2,"score":0.6634258031845093},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.603911280632019},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5739269852638245},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.5697436928749084},{"id":"https://openalex.org/C101468663","wikidata":"https://www.wikidata.org/wiki/Q1620158","display_name":"Modular design","level":2,"score":0.5623379945755005},{"id":"https://openalex.org/C2908586218","wikidata":"https://www.wikidata.org/wiki/Q251","display_name":"Java Programming Language","level":3,"score":0.5299383401870728},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4775315523147583},{"id":"https://openalex.org/C510870499","wikidata":"https://www.wikidata.org/wiki/Q47607","display_name":"SQL","level":2,"score":0.4689454734325409},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4634641706943512},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.45880308747291565},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3520144820213318},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.34285253286361694},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.32785868644714355},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.24793782830238342},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/access.2020.2990980","doi":"https://doi.org/10.1109/access.2020.2990980","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2020.2990980","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/8948470/09079865.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-scientific","cwts-core","doaj","jufo-1","ki-jl-1","norway-1","oarl","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:fabebb77bcf6418a9386a24b275cd1c2","doi":null,"is_oa":true,"landing_page_url":"https://doaj.org/article/fabebb77bcf6418a9386a24b275cd1c2","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 8, Pp 81154-81172 (2020)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1109/access.2020.2990980","doi":"https://doi.org/10.1109/access.2020.2990980","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2020.2990980","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/8948470/09079865.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-scientific","cwts-core","doaj","jufo-1","ki-jl-1","norway-1","oarl","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.8278999924659729,"display_name":"Quality education"}],"study_designs":[],"awards":[{"id":"https://openalex.org/G2008309395","display_name":null,"funder_award_id":"-KK.01.1.1.01","funder_id":"https://openalex.org/F4320335322","funder_display_name":"European Regional Development Fund"},{"id":"https://openalex.org/G2171898597","display_name":null,"funder_award_id":"KK.01.1","funder_id":"https://openalex.org/F4320335322","funder_display_name":"European Regional Development Fund"},{"id":"https://openalex.org/G2699158393","display_name":null,"funder_award_id":"KK.01.1.1.01.0009 (DATACROSS","funder_id":"https://openalex.org/F4320335322","funder_display_name":"European Regional Development Fund"},{"id":"https://openalex.org/G4100563467","display_name":null,"funder_award_id":"KK.01.1.1.01.0009","funder_id":"https://openalex.org/F4320335322","funder_display_name":"European Regional Development Fund"}],"funders":[{"id":"https://openalex.org/F4320335322","display_name":"European Regional Development Fund","ror":"https://ror.org/00k4n6c32"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3022044005.pdf","grobid_xml":"https://content.openalex.org/works/W3022044005.grobid-xml"},"referenced_works_count":53,"referenced_works":["https://openalex.org/W215352090","https://openalex.org/W1509202185","https://openalex.org/W1520017476","https://openalex.org/W1525673018","https://openalex.org/W1545757536","https://openalex.org/W1587378052","https://openalex.org/W1653246633","https://openalex.org/W1965295547","https://openalex.org/W1970459197","https://openalex.org/W1991140109","https://openalex.org/W2000737303","https://openalex.org/W2004496903","https://openalex.org/W2007172362","https://openalex.org/W2009152073","https://openalex.org/W2010775113","https://openalex.org/W2011607672","https://openalex.org/W2020242312","https://openalex.org/W2027591137","https://openalex.org/W2054202879","https://openalex.org/W2070405152","https://openalex.org/W2074194282","https://openalex.org/W2084398551","https://openalex.org/W2105543722","https://openalex.org/W2120411502","https://openalex.org/W2127374659","https://openalex.org/W2130625835","https://openalex.org/W2151828850","https://openalex.org/W2164099431","https://openalex.org/W2186288795","https://openalex.org/W2254299877","https://openalex.org/W2293412016","https://openalex.org/W2295848222","https://openalex.org/W2303032977","https://openalex.org/W2389509314","https://openalex.org/W2399318684","https://openalex.org/W2400080339","https://openalex.org/W2474372315","https://openalex.org/W2501699673","https://openalex.org/W2547115911","https://openalex.org/W2559125818","https://openalex.org/W2765978434","https://openalex.org/W2779408996","https://openalex.org/W2792099437","https://openalex.org/W2886633443","https://openalex.org/W2894540915","https://openalex.org/W2895756513","https://openalex.org/W2921372003","https://openalex.org/W2926052689","https://openalex.org/W2947781045","https://openalex.org/W2951874477","https://openalex.org/W2982347752","https://openalex.org/W4247051014","https://openalex.org/W6608740977"],"related_works":["https://openalex.org/W4382562240","https://openalex.org/W4320728802","https://openalex.org/W4411309568","https://openalex.org/W4321061940","https://openalex.org/W7096478102","https://openalex.org/W4312834620","https://openalex.org/W4391903681","https://openalex.org/W2011607672","https://openalex.org/W2151828850","https://openalex.org/W2736969925"],"abstract_inverted_index":{"Automated":[0],"Programming":[1],"Assessment":[2],"Systems":[3],"(APAS)":[4],"are":[5],"used":[6],"for":[7,135],"overcoming":[8],"problems":[9],"associated":[10],"with":[11,142,245],"manually":[12],"managed":[13],"programming":[14,144,156],"assignments,":[15],"such":[16,55],"as":[17],"objective":[18],"and":[19,25,28,38,43,132,146,165,173,177,195,208,219,239],"efficient":[20],"assessing":[21],"in":[22,40,63,198,231],"large":[23],"classes":[24],"providing":[26],"timely":[27],"helpful":[29],"feedback.":[30],"In":[31],"this":[32,41],"paper":[33],"we":[34],"survey":[35],"the":[36,45,64,76,84,89,110,125,136,217],"literature":[37],"software":[39,92],"field":[42],"identify":[44],"set":[46],"of":[47,78,87,91,112,127,130,179],"necessary":[48],"features":[49],"that":[50,56,203],"make":[51],"APAS":[52,70,107],"comprehensive":[53,69,105],"-":[54],"it":[57,204,240],"can":[58,187,205],"support":[59,226],"all":[60,191],"key":[61],"stages":[62],"assessment":[65],"process.":[66],"Put":[67],"differently,":[68],"is":[71,124,196,230,241],"generic":[72],"enough":[73],"to":[74,95,109,168,210,225],"meet":[75],"demands":[77],"\u201cany\u201d":[79],"computer":[80],"science":[81],"course.":[82],"Despite":[83],"vast":[85],"number":[86],"publications,":[88],"choice":[90],"turns":[93],"out":[94],"be":[96,188,206],"very":[97],"limited.":[98],"We":[99,214],"contribute":[100],"by":[101],"developing":[102],"Edgar,":[103],"a":[104,199,211],"open-source":[106,121],"which,":[108],"best":[111],"our":[113],"knowledge,":[114],"exceeds":[115],"any":[116],"other":[117],"similar":[118],"free":[119],"and/or":[120],"tool.":[122],"Edgar":[123,152,229],"result":[126],"three":[128],"years":[129],"development":[131],"usage":[133],"in,":[134],"time":[137],"being,":[138],"eight":[139],"courses":[140],"dealing":[141],"various":[143,154],"languages":[145],"paradigms":[147],"(C,":[148],"Java,":[149],"SQL,":[150],"etc.).":[151],"supports":[153],"text-based":[155],"languages,":[157],"multi-correct":[158],"multiple-choice":[159],"questions,":[160],"provides":[161],"rich":[162],"exam":[163],"logging":[164],"monitoring":[166],"infrastructure":[167],"prevent":[169],"potential":[170],"fraudulent":[171],"behaviour,":[172],"subsequent":[174],"data":[175,221],"analysis":[176],"visualization":[178],"students'":[180],"scores,":[181],"exams,":[182],"question":[183],"quality,":[184],"etc.":[185],"It":[186],"deployed":[189],"on":[190,216],"major":[192],"operating":[193],"systems":[194],"written":[197],"modular":[200],"fashion":[201],"so":[202],"adjusted":[207],"scaled":[209],"custom":[212],"fit.":[213],"comment":[215],"architecture":[218],"present":[220],"from":[222],"real-world":[223],"use-cases":[224],"these":[227],"claims.":[228],"active":[232],"use":[233],"today":[234],"(1000+":[235],"students":[236],"per":[237],"semester)":[238],"being":[242],"constantly":[243],"developed":[244],"new":[246],"features.":[247]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":18},{"year":2024,"cited_by_count":10},{"year":2023,"cited_by_count":22},{"year":2022,"cited_by_count":7},{"year":2021,"cited_by_count":9},{"year":2020,"cited_by_count":3}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2020-05-13T00:00:00"}