{"id":"https://openalex.org/W2900659377","doi":"https://doi.org/10.1145/3279720.3279729","title":"Automatic Assessment of Source Code Highlighting Tasks","display_name":"Automatic Assessment of Source Code Highlighting Tasks","publication_year":2018,"publication_date":"2018-11-21","ids":{"openalex":"https://openalex.org/W2900659377","doi":"https://doi.org/10.1145/3279720.3279729","mag":"2900659377"},"language":"en","primary_location":{"id":"doi:10.1145/3279720.3279729","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3279720.3279729","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 18th Koli Calling International Conference on Computing Education Research","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102948386","display_name":"Matthias Kr\u00e4mer","orcid":"https://orcid.org/0000-0003-1602-2370"},"institutions":[{"id":"https://openalex.org/I62318514","display_name":"University of Duisburg-Essen","ror":"https://ror.org/04mz5ra38","country_code":"DE","type":"education","lineage":["https://openalex.org/I62318514"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Matthias Kramer","raw_affiliation_strings":["University of Duisburg-Essen, Essen, Germany"],"affiliations":[{"raw_affiliation_string":"University of Duisburg-Essen, Essen, Germany","institution_ids":["https://openalex.org/I62318514"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023210855","display_name":"Mike Barkmin","orcid":null},"institutions":[{"id":"https://openalex.org/I62318514","display_name":"University of Duisburg-Essen","ror":"https://ror.org/04mz5ra38","country_code":"DE","type":"education","lineage":["https://openalex.org/I62318514"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Mike Barkmin","raw_affiliation_strings":["University of Duisburg-Essen, Essen, Germany"],"affiliations":[{"raw_affiliation_string":"University of Duisburg-Essen, Essen, Germany","institution_ids":["https://openalex.org/I62318514"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008053058","display_name":"Torsten Brinda","orcid":"https://orcid.org/0000-0001-6137-4258"},"institutions":[{"id":"https://openalex.org/I62318514","display_name":"University of Duisburg-Essen","ror":"https://ror.org/04mz5ra38","country_code":"DE","type":"education","lineage":["https://openalex.org/I62318514"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Torsten Brinda","raw_affiliation_strings":["University of Duisburg-Essen, Essen, Germany"],"affiliations":[{"raw_affiliation_string":"University of Duisburg-Essen, Essen, Germany","institution_ids":["https://openalex.org/I62318514"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5068267020","display_name":"David Tobinski","orcid":null},"institutions":[{"id":"https://openalex.org/I62318514","display_name":"University of Duisburg-Essen","ror":"https://ror.org/04mz5ra38","country_code":"DE","type":"education","lineage":["https://openalex.org/I62318514"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"David Tobinski","raw_affiliation_strings":["University of Duisburg-Essen, Essen, Germany"],"affiliations":[{"raw_affiliation_string":"University of Duisburg-Essen, Essen, Germany","institution_ids":["https://openalex.org/I62318514"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5102948386"],"corresponding_institution_ids":["https://openalex.org/I62318514"],"apc_list":null,"apc_paid":null,"fwci":0.3927,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.72422642,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":94},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"10"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8149756193161011},{"id":"https://openalex.org/keywords/identifier","display_name":"Identifier","score":0.7073160409927368},{"id":"https://openalex.org/keywords/source-code","display_name":"Source code","score":0.6687270998954773},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6206721067428589},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.5755186080932617},{"id":"https://openalex.org/keywords/class","display_name":"Class (philosophy)","score":0.568436861038208},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5659571290016174},{"id":"https://openalex.org/keywords/object","display_name":"Object (grammar)","score":0.5260329842567444},{"id":"https://openalex.org/keywords/object-code","display_name":"Object code","score":0.4620288014411926},{"id":"https://openalex.org/keywords/binary-number","display_name":"Binary number","score":0.43279531598091125},{"id":"https://openalex.org/keywords/range","display_name":"Range (aeronautics)","score":0.4214167296886444},{"id":"https://openalex.org/keywords/ask-price","display_name":"Ask price","score":0.4108313024044037},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.3898223340511322},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.34893864393234253},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3105694055557251},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.11540481448173523},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.11294090747833252},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.10310810804367065},{"id":"https://openalex.org/keywords/arithmetic","display_name":"Arithmetic","score":0.09304961562156677},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.0685778260231018}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8149756193161011},{"id":"https://openalex.org/C154504017","wikidata":"https://www.wikidata.org/wiki/Q853614","display_name":"Identifier","level":2,"score":0.7073160409927368},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.6687270998954773},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6206721067428589},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5755186080932617},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.568436861038208},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5659571290016174},{"id":"https://openalex.org/C2781238097","wikidata":"https://www.wikidata.org/wiki/Q175026","display_name":"Object (grammar)","level":2,"score":0.5260329842567444},{"id":"https://openalex.org/C154526789","wikidata":"https://www.wikidata.org/wiki/Q157965","display_name":"Object code","level":4,"score":0.4620288014411926},{"id":"https://openalex.org/C48372109","wikidata":"https://www.wikidata.org/wiki/Q3913","display_name":"Binary number","level":2,"score":0.43279531598091125},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.4214167296886444},{"id":"https://openalex.org/C90329073","wikidata":"https://www.wikidata.org/wiki/Q914232","display_name":"Ask price","level":2,"score":0.4108313024044037},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3898223340511322},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.34893864393234253},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3105694055557251},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.11540481448173523},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.11294090747833252},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.10310810804367065},{"id":"https://openalex.org/C94375191","wikidata":"https://www.wikidata.org/wiki/Q11205","display_name":"Arithmetic","level":1,"score":0.09304961562156677},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0685778260231018},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C136264566","wikidata":"https://www.wikidata.org/wiki/Q159810","display_name":"Economy","level":1,"score":0.0},{"id":"https://openalex.org/C133162039","wikidata":"https://www.wikidata.org/wiki/Q1061077","display_name":"Code generation","level":3,"score":0.0},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3279720.3279729","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3279720.3279729","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 18th Koli Calling International Conference on Computing Education Research","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","score":0.6800000071525574,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":23,"referenced_works":["https://openalex.org/W61851215","https://openalex.org/W136732505","https://openalex.org/W1983897914","https://openalex.org/W1987662159","https://openalex.org/W1989492805","https://openalex.org/W2008997252","https://openalex.org/W2014197700","https://openalex.org/W2030644393","https://openalex.org/W2038569843","https://openalex.org/W2038598927","https://openalex.org/W2057136105","https://openalex.org/W2061504941","https://openalex.org/W2093040750","https://openalex.org/W2121473846","https://openalex.org/W2165365113","https://openalex.org/W2290195878","https://openalex.org/W2312700983","https://openalex.org/W2555130386","https://openalex.org/W2786754195","https://openalex.org/W2800615817","https://openalex.org/W4230965853","https://openalex.org/W4231748357","https://openalex.org/W4232362062"],"related_works":["https://openalex.org/W3120493416","https://openalex.org/W2098192829","https://openalex.org/W60611840","https://openalex.org/W2240848044","https://openalex.org/W2332490499","https://openalex.org/W1585130830","https://openalex.org/W170259237","https://openalex.org/W2390085541","https://openalex.org/W2483646612","https://openalex.org/W142712716"],"abstract_inverted_index":{"In":[0],"order":[1],"to":[2,22,38,44,47,50,79,89],"define":[3],"and":[4,46,77,93,113,117],"create":[5],"certain":[6,56],"elements":[7,25],"of":[8,30,54,70,96,103],"object-oriented":[9],"programming":[10],"source":[11,31,42],"code,":[12],"a":[13,27,55,91,145],"necessary":[14],"prerequisite":[15],"for":[16],"prospective":[17],"programmers":[18],"is":[19,36,87],"being":[20],"able":[21],"identify":[23],"these":[24],"in":[26,67,148],"given":[28,146],"piece":[29],"code.":[32],"A":[33],"reasonable":[34],"task":[35],"therefore,":[37],"hand":[39],"out":[40],"existing":[41],"code":[43],"students":[45],"ask":[48],"them":[49],"highlight":[51],"all":[52],"occurrences":[53],"concept,":[57],"e.g.":[58],"class":[59],"identifiers":[60],"or":[61],"method":[62,95],"signatures.":[63],"This":[64],"usually":[65],"results":[66,76],"wide":[68],"range":[69],"highlights.":[71],"To":[72],"quantify":[73],"the":[74],"received":[75],"therefore":[78],"make":[80],"plausible":[81],"inferences":[82],"about":[83],"possible":[84],"abilities,":[85],"it":[86],"vital":[88],"have":[90],"reliable":[92],"valid":[94],"measurement.":[97],"First,":[98],"we":[99],"investigate":[100],"various":[101],"means":[102],"measurement":[104],"towards":[105],"this":[106],"concern,":[107],"including":[108],"measures":[109,123],"from":[110,124],"inter-rater":[111],"reliability":[112],"agreement,":[114],"Cohen's":[115,141],"\u03ba":[116,142],"Krippendorff's":[118],"\u03b1,":[119],"as":[120,122,128],"well":[121],"binary":[125],"classification,":[126],"such":[127],"F1.":[129],"Those":[130],"values":[131],"are":[132],"then":[133],"applied":[134],"on":[135],"constructed":[136],"examples.":[137],"We":[138],"found":[139],"that":[140],"already":[143],"represents":[144],"response":[147],"an":[149],"adequate":[150],"manner.":[151]},"counts_by_year":[{"year":2019,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}