{"id":"https://openalex.org/W3008061209","doi":"https://doi.org/10.1145/3328778.3372650","title":"When Black-box Testing is Not Enough - on Designing Auto-graded Programming Assignments","display_name":"When Black-box Testing is Not Enough - on Designing Auto-graded Programming Assignments","publication_year":2020,"publication_date":"2020-02-25","ids":{"openalex":"https://openalex.org/W3008061209","doi":"https://doi.org/10.1145/3328778.3372650","mag":"3008061209"},"language":"en","primary_location":{"id":"doi:10.1145/3328778.3372650","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3328778.3372650","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 51st ACM Technical Symposium on Computer Science Education","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5112755161","display_name":"Karen H. Jin","orcid":null},"institutions":[{"id":"https://openalex.org/I179093154","display_name":"University of New Hampshire at Manchester","ror":"https://ror.org/04pvpk743","country_code":"US","type":"education","lineage":["https://openalex.org/I179093154"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Karen H. Jin","raw_affiliation_strings":["University of New Hampshire, Manchester, NH, USA"],"affiliations":[{"raw_affiliation_string":"University of New Hampshire, Manchester, NH, USA","institution_ids":["https://openalex.org/I179093154"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5050778054","display_name":"Michel Charpentier","orcid":null},"institutions":[{"id":"https://openalex.org/I161057412","display_name":"University of New Hampshire","ror":"https://ror.org/01rmh9n78","country_code":"US","type":"education","lineage":["https://openalex.org/I161057412"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michel Charpentier","raw_affiliation_strings":["University of New Hampshire, Durham, NH, USA"],"affiliations":[{"raw_affiliation_string":"University of New Hampshire, Durham, NH, USA","institution_ids":["https://openalex.org/I161057412"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5112755161"],"corresponding_institution_ids":["https://openalex.org/I179093154"],"apc_list":null,"apc_paid":null,"fwci":0.2456,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.62123894,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":94},"biblio":{"volume":null,"issue":null,"first_page":"1342","last_page":"1342"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9979000091552734,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.97079998254776,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7886947393417358},{"id":"https://openalex.org/keywords/white-box-testing","display_name":"White-box testing","score":0.6629509925842285},{"id":"https://openalex.org/keywords/grading","display_name":"Grading (engineering)","score":0.6341555118560791},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.630560040473938},{"id":"https://openalex.org/keywords/black-box","display_name":"Black box","score":0.45337292551994324},{"id":"https://openalex.org/keywords/functional-specification","display_name":"Functional specification","score":0.42779237031936646},{"id":"https://openalex.org/keywords/functional-programming","display_name":"Functional programming","score":0.4179005026817322},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.39189404249191284},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2548828125},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.17564299702644348},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.09783801436424255},{"id":"https://openalex.org/keywords/software-construction","display_name":"Software construction","score":0.09357896447181702}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7886947393417358},{"id":"https://openalex.org/C162443782","wikidata":"https://www.wikidata.org/wiki/Q1066228","display_name":"White-box testing","level":5,"score":0.6629509925842285},{"id":"https://openalex.org/C2777286243","wikidata":"https://www.wikidata.org/wiki/Q5591926","display_name":"Grading (engineering)","level":2,"score":0.6341555118560791},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.630560040473938},{"id":"https://openalex.org/C94966114","wikidata":"https://www.wikidata.org/wiki/Q29256","display_name":"Black box","level":2,"score":0.45337292551994324},{"id":"https://openalex.org/C57371142","wikidata":"https://www.wikidata.org/wiki/Q16914225","display_name":"Functional specification","level":5,"score":0.42779237031936646},{"id":"https://openalex.org/C42383842","wikidata":"https://www.wikidata.org/wiki/Q193076","display_name":"Functional programming","level":2,"score":0.4179005026817322},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.39189404249191284},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2548828125},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.17564299702644348},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.09783801436424255},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.09357896447181702},{"id":"https://openalex.org/C147176958","wikidata":"https://www.wikidata.org/wiki/Q77590","display_name":"Civil engineering","level":1,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3328778.3372650","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3328778.3372650","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 51st ACM Technical Symposium on Computer Science Education","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.8100000023841858,"display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W1501213200","https://openalex.org/W112492789","https://openalex.org/W1532322220","https://openalex.org/W2131714870","https://openalex.org/W4242844989","https://openalex.org/W2171640008","https://openalex.org/W4319452366","https://openalex.org/W2543388663","https://openalex.org/W89376891","https://openalex.org/W2041668232"],"abstract_inverted_index":{"Automatic":[0],"programming":[1,61,79,104],"assignment":[2,116,160],"assessment":[3,96],"is":[4,97,101,131],"often":[5],"premised":[6],"on":[7,16,35,103,182],"black-box":[8,153],"testing.":[9,123],"Grading":[10],"of":[11,22,114,126,145,157,204],"student":[12,173],"submissions":[13],"typically":[14],"relies":[15],"functional":[17,49,60,150],"specifications":[18,135,151],"expressed":[19],"in":[20,107,209],"terms":[21],"expected":[23],"outputs":[24],"for":[25,51],"given":[26],"test":[27,176],"inputs.":[28],"Many":[29],"upper-level":[30],"courses,":[31],"however,":[32],"are":[33,42,67,74,207],"centered":[34,102],"concepts":[36],"that":[37,47,57,111,132,185],"relate":[38],"to":[39,76,94,148,193],"how":[40],"programs":[41],"implemented.":[43],"In":[44],"a":[45,91,109,141,158,168],"course":[46],"teaches":[48],"programming,":[50],"instance,":[52],"the":[53,70,112,115,134,137],"assignments":[54,105,130],"should":[55,178],"require":[56],"students":[58,73],"use":[59,77],"techniques,":[62],"even":[63],"if":[64],"imperative":[65],"solutions":[66],"supported":[68],"by":[69],"language.":[71],"When":[72],"required":[75],"certain":[78,169],"language":[80,198],"constructs,":[81],"algorithms":[82,196],"or":[83,171,197],"design":[84],"strategies":[85],"as":[86],"they":[87],"implement":[88],"their":[89],"programs,":[90],"different":[92],"approach":[93,206],"automated":[95,122],"needed.":[98],"Our":[99],"strategy":[100],"designed":[106],"such":[108,128],"way":[110],"internals":[113],"implementation":[117,183],"can":[118],"be":[119,191],"evaluated":[120],"through":[121],"A":[124],"challenge":[125],"designing":[127],"auto-graded":[129],"both":[133],"and":[136,152,202],"grading":[138],"tests":[139],"have":[140,162],"much":[142],"higher":[143],"level":[144],"complexity":[146],"compared":[147],"plain":[149],"tests.":[154],"The":[155,200],"specification":[156],"homework":[159],"must":[161,190],"precise":[163],"requirements,":[164],"but":[165,189],"not":[166,179,187],"prescribe":[167],"solution":[170],"impair":[172],"creativity.":[174],"Furthermore,":[175],"cases":[177],"inadvertently":[180],"rely":[181],"details":[184],"were":[186],"specified,":[188],"able":[192],"detect":[194],"forbidden":[195],"features.":[199],"benefits":[201],"difficulties":[203],"our":[205],"discussed":[208],"this":[210],"work.":[211]},"counts_by_year":[{"year":2023,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}