{"id":"https://openalex.org/W4406677385","doi":"https://doi.org/10.1145/3702163.3702169","title":"Automatic conversational assessment using large language model technology","display_name":"Automatic conversational assessment using large language model technology","publication_year":2024,"publication_date":"2024-09-18","ids":{"openalex":"https://openalex.org/W4406677385","doi":"https://doi.org/10.1145/3702163.3702169"},"language":"en","primary_location":{"id":"doi:10.1145/3702163.3702169","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3702163.3702169","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 16th International Conference on Education Technology and Computers","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5052863099","display_name":"Jan Bergerhoff","orcid":"https://orcid.org/0000-0001-9720-1340"},"institutions":[{"id":"https://openalex.org/I34352273","display_name":"Maastricht University","ror":"https://ror.org/02jz4aj89","country_code":"NL","type":"education","lineage":["https://openalex.org/I34352273"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Jan Bergerhoff","raw_affiliation_strings":["Department of Advanced Computing Sciences, Maastricht University, Maastricht, Limburg, NL"],"affiliations":[{"raw_affiliation_string":"Department of Advanced Computing Sciences, Maastricht University, Maastricht, Limburg, NL","institution_ids":["https://openalex.org/I34352273"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115972242","display_name":"Johannes Bendler","orcid":"https://orcid.org/0009-0002-3568-5145"},"institutions":[{"id":"https://openalex.org/I34352273","display_name":"Maastricht University","ror":"https://ror.org/02jz4aj89","country_code":"NL","type":"education","lineage":["https://openalex.org/I34352273"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Johannes Bendler","raw_affiliation_strings":["Department of Advanced Computing Sciences, Maastricht University, Maastricht, Limburg, NL"],"affiliations":[{"raw_affiliation_string":"Department of Advanced Computing Sciences, Maastricht University, Maastricht, Limburg, NL","institution_ids":["https://openalex.org/I34352273"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003375445","display_name":"\u0421\u0442\u0435\u0444\u0430\u043d \u0421\u0442\u0435\u0444\u0430\u043d\u043e\u0432","orcid":null},"institutions":[{"id":"https://openalex.org/I34352273","display_name":"Maastricht University","ror":"https://ror.org/02jz4aj89","country_code":"NL","type":"education","lineage":["https://openalex.org/I34352273"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Stefan Stefanov","raw_affiliation_strings":["Department of Advanced Computing Sciences, Maastricht University, Maastricht, Limburg, NL"],"affiliations":[{"raw_affiliation_string":"Department of Advanced Computing Sciences, Maastricht University, Maastricht, Limburg, NL","institution_ids":["https://openalex.org/I34352273"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115972243","display_name":"Enrico Cavinato","orcid":"https://orcid.org/0009-0009-5761-0903"},"institutions":[{"id":"https://openalex.org/I34352273","display_name":"Maastricht University","ror":"https://ror.org/02jz4aj89","country_code":"NL","type":"education","lineage":["https://openalex.org/I34352273"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Enrico Cavinato","raw_affiliation_strings":["Department of Advanced Computing Sciences, Maastricht University, Maastricht, Limburg, NL"],"affiliations":[{"raw_affiliation_string":"Department of Advanced Computing Sciences, Maastricht University, Maastricht, Limburg, NL","institution_ids":["https://openalex.org/I34352273"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111559664","display_name":"L.J.M. Esser","orcid":null},"institutions":[{"id":"https://openalex.org/I34352273","display_name":"Maastricht University","ror":"https://ror.org/02jz4aj89","country_code":"NL","type":"education","lineage":["https://openalex.org/I34352273"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Leonard Esser","raw_affiliation_strings":["Department of Advanced Computing Sciences, Maastricht University, Maastricht, Limburg, NL"],"affiliations":[{"raw_affiliation_string":"Department of Advanced Computing Sciences, Maastricht University, Maastricht, Limburg, NL","institution_ids":["https://openalex.org/I34352273"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108277389","display_name":"T.T. Tran","orcid":"https://orcid.org/0009-0000-3183-4006"},"institutions":[{"id":"https://openalex.org/I34352273","display_name":"Maastricht University","ror":"https://ror.org/02jz4aj89","country_code":"NL","type":"education","lineage":["https://openalex.org/I34352273"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Tommy Tran","raw_affiliation_strings":["Department of Advanced Computing Sciences, Maastricht University, Maastricht, Limburg, NL"],"affiliations":[{"raw_affiliation_string":"Department of Advanced Computing Sciences, Maastricht University, Maastricht, Limburg, NL","institution_ids":["https://openalex.org/I34352273"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5105460983","display_name":"Aki H\u00e4rm\u00e4","orcid":"https://orcid.org/0000-0002-2966-3305"},"institutions":[{"id":"https://openalex.org/I34352273","display_name":"Maastricht University","ror":"https://ror.org/02jz4aj89","country_code":"NL","type":"education","lineage":["https://openalex.org/I34352273"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Aki H\u00e4rm\u00e4","raw_affiliation_strings":["Department of Advanced Computing Sciences, Maastricht University, Maastricht, Limburg, NL"],"affiliations":[{"raw_affiliation_string":"Department of Advanced Computing Sciences, Maastricht University, Maastricht, Limburg, NL","institution_ids":["https://openalex.org/I34352273"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5052863099"],"corresponding_institution_ids":["https://openalex.org/I34352273"],"apc_list":null,"apc_paid":null,"fwci":1.5783,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.88926283,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"39","last_page":"45"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9164000153541565,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9164000153541565,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7914852499961853},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5182921290397644},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.433709055185318},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.35518038272857666},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.34591183066368103}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7914852499961853},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5182921290397644},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.433709055185318},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.35518038272857666},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.34591183066368103}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1145/3702163.3702169","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3702163.3702169","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 16th International Conference on Education Technology and Computers","raw_type":"proceedings-article"},{"id":"pmh:oai:cris.maastrichtuniversity.nl:publications/60bab9f0-1f8f-455f-962c-0323faa87074","is_oa":false,"landing_page_url":"https://cris.maastrichtuniversity.nl/en/publications/60bab9f0-1f8f-455f-962c-0323faa87074","pdf_url":null,"source":{"id":"https://openalex.org/S4306402616","display_name":"Research Publications (Maastricht University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I34352273","host_organization_name":"Maastricht University","host_organization_lineage":["https://openalex.org/I34352273"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Bergerhoff, J, Bendler, J, Stefanov, S, Cavinato, E, Esser, L, Tran, T & H\u00e4rm\u00e4, A 2025, Automatic conversational assessment using large language model technology. in Proceedings of the 2024 the 16th International Conference on Education Technology and Computers, ICETC 2024. Association for Computing Machinery, New York, NY, USA, Proceedings of the International Conference on Education Technology and Computers, pp. 39-45. https://doi.org/10.1145/3702163.3702169","raw_type":"info:eu-repo/semantics/publishedVersion"},{"id":"pmh:oai:research.vu.nl:openaire_cris_publications/d6fbcb21-60f5-4133-9dc4-1d737130adab","is_oa":false,"landing_page_url":"https://hdl.handle.net/1871.1/d6fbcb21-60f5-4133-9dc4-1d737130adab","pdf_url":null,"source":{"id":"https://openalex.org/S4306401107","display_name":"VU Research Portal","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I865915315","host_organization_name":"Vrije Universiteit Amsterdam","host_organization_lineage":["https://openalex.org/I865915315"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Bergerhoff, J, Bendler, J, Stefanov, S, Cavinato, E, Esser, L, Tran, T & H\u00e4rm\u00e4, A 2024, Automatic conversational assessment using large language model technology. in Automatic conversational assessment using large language model technology. pp. 39-45. https://doi.org/10.1145/3702163.3702169","raw_type":"contributionToPeriodical"},{"id":"pmh:oai:research.vu.nl:publications/d6fbcb21-60f5-4133-9dc4-1d737130adab","is_oa":false,"landing_page_url":"https://research.vu.nl/en/publications/d6fbcb21-60f5-4133-9dc4-1d737130adab","pdf_url":null,"source":{"id":"https://openalex.org/S4306401107","display_name":"VU Research Portal","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I865915315","host_organization_name":"Vrije Universiteit Amsterdam","host_organization_lineage":["https://openalex.org/I865915315"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Bergerhoff, J, Bendler, J, Stefanov, S, Cavinato, E, Esser, L, Tran, T & H\u00e4rm\u00e4, A 2024, Automatic conversational assessment using large language model technology. in Automatic conversational assessment using large language model technology. pp. 39-45. https://doi.org/10.1145/3702163.3702169","raw_type":"contributionToPeriodical"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.7900000214576721,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":25,"referenced_works":["https://openalex.org/W1967729042","https://openalex.org/W2803087880","https://openalex.org/W2930638636","https://openalex.org/W3007294660","https://openalex.org/W3033346189","https://openalex.org/W3154168950","https://openalex.org/W3167554351","https://openalex.org/W3197767417","https://openalex.org/W3216314518","https://openalex.org/W4234918932","https://openalex.org/W4292779060","https://openalex.org/W4320003957","https://openalex.org/W4360891295","https://openalex.org/W4366403195","https://openalex.org/W4378902339","https://openalex.org/W4384200891","https://openalex.org/W4385507096","https://openalex.org/W4385878593","https://openalex.org/W4386836600","https://openalex.org/W4387585613","https://openalex.org/W4389228739","https://openalex.org/W4390897279","https://openalex.org/W4390897568","https://openalex.org/W4392624992","https://openalex.org/W4394826750"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Student":[0],"evaluation":[1],"is":[2],"an":[3],"important,":[4],"yet":[5],"costly,":[6],"part":[7],"of":[8,57,91,95],"instruction.":[9],"Traditional":[10],"exams":[11],"are":[12],"a":[13,24,32,40,60,64,80,85,92,101],"burden":[14],"for":[15,19,34],"teachers":[16],"and":[17,46,82,115],"stressful":[18],"students.":[20,98],"This":[21],"paper":[22],"uses":[23],"large":[25],"language":[26],"model":[27],"(LLM)":[28],"technology":[29],"to":[30,53,112],"create":[31],"system":[33,72],"Automated":[35],"Conversational":[36],"Assessment,":[37],"ACA,":[38],"where":[39],"dialog":[41],"system,":[42],"based":[43],"on":[44,100],"content":[45],"intended":[47],"learning":[48],"outcomes,":[49],"interviews":[50],"the":[51,55,70,76,89,96,104,108],"student":[52],"determine":[54],"level":[56],"learning.":[58],"In":[59],"pilot":[61],"experiment":[62],"in":[63],"university":[65],"course,":[66],"we":[67],"found":[68],"that":[69,107],"ACA":[71],"scores":[73],"correlate":[74],"with":[75,88],"grades":[77],"given":[78],"by":[79],"human":[81],"also":[83],"have":[84],"positive":[86],"correlation":[87],"results":[90],"conventional":[93],"exam":[94],"same":[97],"Based":[99],"questionnaire":[102],"study,":[103],"students":[105],"responded":[106],"assessment":[109],"was":[110],"perceived":[111],"be":[113],"fair":[114],"acceptable.":[116]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1}],"updated_date":"2026-03-24T08:02:53.985720","created_date":"2025-10-10T00:00:00"}