{"id":"https://openalex.org/W4400336055","doi":"https://doi.org/10.1145/3649405.3659514","title":"LLMs in Open and Closed Book Examinations in a Final Year Applied Machine Learning Course (Early Findings)","display_name":"LLMs in Open and Closed Book Examinations in a Final Year Applied Machine Learning Course (Early Findings)","publication_year":2024,"publication_date":"2024-07-04","ids":{"openalex":"https://openalex.org/W4400336055","doi":"https://doi.org/10.1145/3649405.3659514"},"language":"en","primary_location":{"id":"doi:10.1145/3649405.3659514","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3649405.3659514","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 on Innovation and Technology in Computer Science Education V. 2","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5008927493","display_name":"Keith Quille","orcid":"https://orcid.org/0000-0002-1414-5142"},"institutions":[{"id":"https://openalex.org/I4210144925","display_name":"Technological University Dublin","ror":"https://ror.org/04t0qbt32","country_code":"IE","type":"education","lineage":["https://openalex.org/I4210144925"]}],"countries":["IE"],"is_corresponding":false,"raw_author_name":"Keith Quille","raw_affiliation_strings":["School of ECDT, TU Dublin, Dublin, Ireland"],"raw_orcid":"https://orcid.org/0000-0002-1414-5142","affiliations":[{"raw_affiliation_string":"School of ECDT, TU Dublin, Dublin, Ireland","institution_ids":["https://openalex.org/I4210144925"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054692443","display_name":"Brett A. Becker","orcid":"https://orcid.org/0000-0003-1446-647X"},"institutions":[{"id":"https://openalex.org/I100930933","display_name":"University College Dublin","ror":"https://ror.org/05m7pjf47","country_code":"IE","type":"education","lineage":["https://openalex.org/I100930933"]}],"countries":["IE"],"is_corresponding":false,"raw_author_name":"Brett A. Becker","raw_affiliation_strings":["School of Computer Science, UCD, Dublin, Ireland"],"raw_orcid":"https://orcid.org/0000-0003-1446-647X","affiliations":[{"raw_affiliation_string":"School of Computer Science, UCD, Dublin, Ireland","institution_ids":["https://openalex.org/I100930933"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071612410","display_name":"R\u00f3is\u00edn Faherty","orcid":"https://orcid.org/0000-0002-3906-1500"},"institutions":[{"id":"https://openalex.org/I4210144925","display_name":"Technological University Dublin","ror":"https://ror.org/04t0qbt32","country_code":"IE","type":"education","lineage":["https://openalex.org/I4210144925"]}],"countries":["IE"],"is_corresponding":false,"raw_author_name":"Roisin Faherty","raw_affiliation_strings":["School of ECDT, TU Dublin, Dublin, Ireland"],"raw_orcid":"https://orcid.org/0000-0002-3906-1500","affiliations":[{"raw_affiliation_string":"School of ECDT, TU Dublin, Dublin, Ireland","institution_ids":["https://openalex.org/I4210144925"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108951992","display_name":"Damien Gordon","orcid":null},"institutions":[{"id":"https://openalex.org/I4210144925","display_name":"Technological University Dublin","ror":"https://ror.org/04t0qbt32","country_code":"IE","type":"education","lineage":["https://openalex.org/I4210144925"]}],"countries":["IE"],"is_corresponding":false,"raw_author_name":"Damien Gordon","raw_affiliation_strings":["School of CS, TU Dublin, Dublin, Ireland"],"raw_orcid":"https://orcid.org/0000-0002-3875-4065","affiliations":[{"raw_affiliation_string":"School of CS, TU Dublin, Dublin, Ireland","institution_ids":["https://openalex.org/I4210144925"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024713741","display_name":"Miriam Harte","orcid":"https://orcid.org/0009-0007-9334-0416"},"institutions":[{"id":"https://openalex.org/I4210144925","display_name":"Technological University Dublin","ror":"https://ror.org/04t0qbt32","country_code":"IE","type":"education","lineage":["https://openalex.org/I4210144925"]}],"countries":["IE"],"is_corresponding":false,"raw_author_name":"Miriam Harte","raw_affiliation_strings":["School of ECDT, TU Dublin, Dublin, Ireland"],"raw_orcid":"https://orcid.org/0009-0007-9334-0416","affiliations":[{"raw_affiliation_string":"School of ECDT, TU Dublin, Dublin, Ireland","institution_ids":["https://openalex.org/I4210144925"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074715374","display_name":"Svetlana Hensman","orcid":"https://orcid.org/0000-0002-1804-2925"},"institutions":[{"id":"https://openalex.org/I4210144925","display_name":"Technological University Dublin","ror":"https://ror.org/04t0qbt32","country_code":"IE","type":"education","lineage":["https://openalex.org/I4210144925"]}],"countries":["IE"],"is_corresponding":false,"raw_author_name":"Svetlana Hensman","raw_affiliation_strings":["School of CS, TU Dublin, Dublin, Ireland"],"raw_orcid":"https://orcid.org/0000-0002-1804-2925","affiliations":[{"raw_affiliation_string":"School of CS, TU Dublin, Dublin, Ireland","institution_ids":["https://openalex.org/I4210144925"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036527230","display_name":"Markus Hofmann","orcid":"https://orcid.org/0000-0002-9215-8862"},"institutions":[{"id":"https://openalex.org/I4210144925","display_name":"Technological University Dublin","ror":"https://ror.org/04t0qbt32","country_code":"IE","type":"education","lineage":["https://openalex.org/I4210144925"]}],"countries":["IE"],"is_corresponding":false,"raw_author_name":"Markus Hofmann","raw_affiliation_strings":["School of C&I, TU Dublin, Dublin, Ireland"],"raw_orcid":"https://orcid.org/0000-0002-9215-8862","affiliations":[{"raw_affiliation_string":"School of C&I, TU Dublin, Dublin, Ireland","institution_ids":["https://openalex.org/I4210144925"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030123297","display_name":"Keith Nolan","orcid":"https://orcid.org/0000-0001-7974-4253"},"institutions":[{"id":"https://openalex.org/I4210144925","display_name":"Technological University Dublin","ror":"https://ror.org/04t0qbt32","country_code":"IE","type":"education","lineage":["https://openalex.org/I4210144925"]}],"countries":["IE"],"is_corresponding":false,"raw_author_name":"Keith Nolan","raw_affiliation_strings":["School of ECDT, TU Dublin, Dublin, Ireland"],"raw_orcid":"https://orcid.org/0000-0001-7974-4253","affiliations":[{"raw_affiliation_string":"School of ECDT, TU Dublin, Dublin, Ireland","institution_ids":["https://openalex.org/I4210144925"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5079941848","display_name":"Ciar\u00e1n O\u2019Leary","orcid":"https://orcid.org/0000-0002-0070-6420"},"institutions":[{"id":"https://openalex.org/I4210144925","display_name":"Technological University Dublin","ror":"https://ror.org/04t0qbt32","country_code":"IE","type":"education","lineage":["https://openalex.org/I4210144925"]}],"countries":["IE"],"is_corresponding":false,"raw_author_name":"Ciaran O'Leary","raw_affiliation_strings":["Faculty of CDD, TU Dublin, Dublin, Ireland"],"raw_orcid":"https://orcid.org/0000-0002-0070-6420","affiliations":[{"raw_affiliation_string":"Faculty of CDD, TU Dublin, Dublin, Ireland","institution_ids":["https://openalex.org/I4210144925"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":9,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.11778313,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"822","last_page":"822"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13523","display_name":"Mathematics, Computing, and Information Processing","score":0.9103000164031982,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13523","display_name":"Mathematics, Computing, and Information Processing","score":0.9103000164031982,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12171","display_name":"Open Education and E-Learning","score":0.9021999835968018,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/course","display_name":"Course (navigation)","score":0.8128911256790161},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5153850317001343},{"id":"https://openalex.org/keywords/short-course","display_name":"Short course","score":0.43879976868629456},{"id":"https://openalex.org/keywords/mathematics-education","display_name":"Mathematics education","score":0.4258127510547638},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3479195833206177},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.3001348376274109},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.2212902307510376},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.14183688163757324},{"id":"https://openalex.org/keywords/pediatrics","display_name":"Pediatrics","score":0.07526588439941406}],"concepts":[{"id":"https://openalex.org/C2777552389","wikidata":"https://www.wikidata.org/wiki/Q1962728","display_name":"Course (navigation)","level":2,"score":0.8128911256790161},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5153850317001343},{"id":"https://openalex.org/C2780159726","wikidata":"https://www.wikidata.org/wiki/Q4008851","display_name":"Short course","level":2,"score":0.43879976868629456},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.4258127510547638},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3479195833206177},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.3001348376274109},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2212902307510376},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.14183688163757324},{"id":"https://openalex.org/C187212893","wikidata":"https://www.wikidata.org/wiki/Q123028","display_name":"Pediatrics","level":1,"score":0.07526588439941406},{"id":"https://openalex.org/C146978453","wikidata":"https://www.wikidata.org/wiki/Q3798668","display_name":"Aerospace engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3649405.3659514","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3649405.3659514","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 on Innovation and Technology in Computer Science Education V. 2","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.7300000190734863,"display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W265949992","https://openalex.org/W2228453512","https://openalex.org/W2185740551","https://openalex.org/W2306280204","https://openalex.org/W4388030826","https://openalex.org/W4250761919","https://openalex.org/W4245799793","https://openalex.org/W3045286878","https://openalex.org/W2760472119","https://openalex.org/W2274929579"],"abstract_inverted_index":{"This":[0,66,118],"research":[1,79,123],"has":[2],"three":[3],"prongs,":[4],"with":[5,109,112],"each":[6],"comparing":[7,40],"open-":[8],"and":[9,39,42,54,61,102,131],"closed-book":[10],"exam":[11],"questions":[12,53],"across":[13],"six":[14,132],"years":[15],"(2017-2023)":[16],"in":[17,75,88],"a":[18,48,121],"final":[19],"year":[20],"undergraduate":[21],"applied":[22],"machine":[23],"learning":[24],"course.":[25],"First,":[26],"the":[27,30,51,76],"authors":[28],"evaluated":[29],"performance":[31],"of":[32,68],"numerous":[33],"LLMs,":[34],"compared":[35],"to":[36],"student":[37],"performance,":[38],"open":[41],"closed":[43],"book":[44],"exams.":[45],"Second,":[46],"at":[47],"micro":[49],"level,":[50],"examination":[52],"categories":[55],"for":[56],"which":[57],"LLMs":[58],"were":[59,64],"most":[60],"least":[62],"effective":[63],"compared.":[65],"level":[67],"analysis":[69],"is":[70,120],"rarely":[71],"if":[72],"ever,":[73],"discussed":[74],"literature.":[77],"The":[78],"finally":[80],"investigates":[81],"LLM":[82,100,103],"detection":[83],"techniques,":[84],"specifically":[85],"their":[86],"efficacy":[87],"identifying":[89],"replies":[90],"created":[91],"wholly":[92],"by":[93,110],"an":[94,113],"LLM.":[95],"It":[96],"considers":[97],"both":[98],"raw":[99],"outputs":[101,104],"that":[105],"have":[106],"been":[107],"tampered":[108],"students,":[111],"emphasis":[114],"on":[115],"academic":[116,129],"integrity.":[117],"study":[119],"staff-student":[122],"collaboration,":[124],"featuring":[125],"contributions":[126],"from":[127],"eight":[128],"professionals":[130],"students.":[133]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}