{"id":"https://openalex.org/W2730709268","doi":"https://doi.org/10.1145/3059009.3059051","title":"Evaluating Test Suite Effectiveness and Assessing Student Code via Constraint Logic Programming","display_name":"Evaluating Test Suite Effectiveness and Assessing Student Code via Constraint Logic Programming","publication_year":2017,"publication_date":"2017-06-28","ids":{"openalex":"https://openalex.org/W2730709268","doi":"https://doi.org/10.1145/3059009.3059051","mag":"2730709268"},"language":"en","primary_location":{"id":"doi:10.1145/3059009.3059051","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3059009.3059051","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3059009.3059051","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2017 ACM Conference on Innovation and Technology in Computer Science Education","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3059009.3059051","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5059923669","display_name":"Kyle Dewey","orcid":"https://orcid.org/0000-0002-7998-3104"},"institutions":[{"id":"https://openalex.org/I154570441","display_name":"University of California, Santa Barbara","ror":"https://ror.org/02t274463","country_code":"US","type":"education","lineage":["https://openalex.org/I154570441"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Kyle Dewey","raw_affiliation_strings":["University of California, Santa Barbara, Santa Barbara, CA, USA"],"affiliations":[{"raw_affiliation_string":"University of California, Santa Barbara, Santa Barbara, CA, USA","institution_ids":["https://openalex.org/I154570441"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075549720","display_name":"Phillip Conrad","orcid":"https://orcid.org/0000-0003-3676-7775"},"institutions":[{"id":"https://openalex.org/I154570441","display_name":"University of California, Santa Barbara","ror":"https://ror.org/02t274463","country_code":"US","type":"education","lineage":["https://openalex.org/I154570441"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Phillip Conrad","raw_affiliation_strings":["University of California, Santa Barbara, Santa Barbara, CA, USA"],"affiliations":[{"raw_affiliation_string":"University of California, Santa Barbara, Santa Barbara, CA, USA","institution_ids":["https://openalex.org/I154570441"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076439311","display_name":"Michelle Craig","orcid":"https://orcid.org/0000-0001-8283-0072"},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Michelle Craig","raw_affiliation_strings":["University of Toronto, Toronto, ON, Canada"],"affiliations":[{"raw_affiliation_string":"University of Toronto, Toronto, ON, Canada","institution_ids":["https://openalex.org/I185261750"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5112294959","display_name":"Elena Morozova","orcid":null},"institutions":[{"id":"https://openalex.org/I154570441","display_name":"University of California, Santa Barbara","ror":"https://ror.org/02t274463","country_code":"US","type":"education","lineage":["https://openalex.org/I154570441"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Elena Morozova","raw_affiliation_strings":["University of California, Santa Barbara, Santa Barbara, CA, USA"],"affiliations":[{"raw_affiliation_string":"University of California, Santa Barbara, Santa Barbara, CA, USA","institution_ids":["https://openalex.org/I154570441"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5059923669"],"corresponding_institution_ids":["https://openalex.org/I154570441"],"apc_list":null,"apc_paid":null,"fwci":1.8263,"has_fulltext":true,"cited_by_count":9,"citation_normalized_percentile":{"value":0.85696068,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"317","last_page":"322"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/test-suite","display_name":"Test suite","score":0.9265228509902954},{"id":"https://openalex.org/keywords/suite","display_name":"Suite","score":0.8631758689880371},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7899104356765747},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.63758784532547},{"id":"https://openalex.org/keywords/constraint","display_name":"Constraint (computer-aided design)","score":0.5737192034721375},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.5414495468139648},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.4855653941631317},{"id":"https://openalex.org/keywords/test-case","display_name":"Test case","score":0.4769096374511719},{"id":"https://openalex.org/keywords/constraint-programming","display_name":"Constraint programming","score":0.4761222004890442},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.3900737166404724},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.1961345374584198},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.08289849758148193},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.0780630111694336}],"concepts":[{"id":"https://openalex.org/C151552104","wikidata":"https://www.wikidata.org/wiki/Q7705809","display_name":"Test suite","level":4,"score":0.9265228509902954},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.8631758689880371},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7899104356765747},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.63758784532547},{"id":"https://openalex.org/C2776036281","wikidata":"https://www.wikidata.org/wiki/Q48769818","display_name":"Constraint (computer-aided design)","level":2,"score":0.5737192034721375},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.5414495468139648},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4855653941631317},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.4769096374511719},{"id":"https://openalex.org/C173404611","wikidata":"https://www.wikidata.org/wiki/Q528588","display_name":"Constraint programming","level":3,"score":0.4761222004890442},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3900737166404724},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.1961345374584198},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.08289849758148193},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0780630111694336},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C137631369","wikidata":"https://www.wikidata.org/wiki/Q7617831","display_name":"Stochastic programming","level":2,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3059009.3059051","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3059009.3059051","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3059009.3059051","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2017 ACM Conference on Innovation and Technology in Computer Science Education","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3059009.3059051","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3059009.3059051","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3059009.3059051","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2017 ACM Conference on Innovation and Technology in Computer Science Education","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.7900000214576721}],"awards":[{"id":"https://openalex.org/G794432730","display_name":null,"funder_award_id":"CCF-1319060","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2730709268.pdf","grobid_xml":"https://content.openalex.org/works/W2730709268.grobid-xml"},"referenced_works_count":20,"referenced_works":["https://openalex.org/W61686890","https://openalex.org/W1531203382","https://openalex.org/W1969084508","https://openalex.org/W1984762903","https://openalex.org/W2012312630","https://openalex.org/W2030367896","https://openalex.org/W2034797403","https://openalex.org/W2040937391","https://openalex.org/W2052247101","https://openalex.org/W2098456636","https://openalex.org/W2111295912","https://openalex.org/W2123396057","https://openalex.org/W2151828850","https://openalex.org/W2156723666","https://openalex.org/W2162055787","https://openalex.org/W2171480813","https://openalex.org/W2249862487","https://openalex.org/W2911601159","https://openalex.org/W3003163926","https://openalex.org/W4238083723"],"related_works":["https://openalex.org/W1485837041","https://openalex.org/W3204955359","https://openalex.org/W4291309325","https://openalex.org/W2950474130","https://openalex.org/W1598160211","https://openalex.org/W2179621094","https://openalex.org/W2737452945","https://openalex.org/W4313469555","https://openalex.org/W2067499928","https://openalex.org/W1547011777"],"abstract_inverted_index":{"A":[0],"good":[1],"suite":[2,55,68,103,113],"of":[3,16,65],"test":[4,29,54,67,102],"inputs":[5],"is":[6,32],"an":[7,89],"indispensable":[8],"tool":[9],"both":[10],"for":[11,50,88],"manual":[12],"and":[13,80],"automated":[14],"assessment":[15],"student":[17,86],"submissions":[18,87],"to":[19,26,34,60,106],"programming":[20,94],"assignments.":[21],"Yet,":[22],"without":[23],"a":[24,48,52,66,82,92,100],"way":[25],"evaluate":[27],"our":[28,44,78],"suites,":[30],"it":[31],"difficult":[33],"know":[35],"how":[36],"well":[37],"we":[38],"are":[39],"doing,":[40],"much":[41],"less":[42],"improve":[43],"practice.":[45],"We":[46,76],"present":[47,81],"technique":[49,79],"evaluating":[51],"hand-generated":[53],"by":[56],"comparing":[57],"its":[58],"ability":[59],"find":[61],"defects":[62,109],"against":[63],"that":[64,99,110],"generated":[69],"automatically":[70],"using":[71,85],"Constraint":[72],"Logic":[73],"Programming":[74],"(CLP).":[75],"describe":[77],"case":[83],"study":[84],"assignment":[90],"from":[91],"second-year":[93],"course.":[95],"Our":[96],"results":[97],"show":[98],"CLP-generated":[101],"was":[104],"able":[105],"identify":[107],"significant":[108],"the":[111],"instructor-generated":[112],"missed,":[114],"despite":[115],"having":[116],"similar":[117],"code":[118],"coverage.":[119]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":3},{"year":2019,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}