{"id":"https://openalex.org/W4416863255","doi":"https://doi.org/10.1109/cw68232.2025.00059","title":"Prompt-and-Check: Using Large Language Models to Evaluate Communication Protocol Compliance in Simulation-Based Training","display_name":"Prompt-and-Check: Using Large Language Models to Evaluate Communication Protocol Compliance in Simulation-Based Training","publication_year":2025,"publication_date":"2025-10-14","ids":{"openalex":"https://openalex.org/W4416863255","doi":"https://doi.org/10.1109/cw68232.2025.00059"},"language":"en","primary_location":{"id":"doi:10.1109/cw68232.2025.00059","is_oa":false,"landing_page_url":"https://doi.org/10.1109/cw68232.2025.00059","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 International Conference on Cyberworlds (CW\uff09","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2508.08652","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5120623758","display_name":"Vishakha Lall","orcid":null},"institutions":[{"id":"https://openalex.org/I202502355","display_name":"Singapore Polytechnic","ror":"https://ror.org/05pxcn248","country_code":"SG","type":"education","lineage":["https://openalex.org/I202502355"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Vishakha Lall","raw_affiliation_strings":["Singapore Polytechnic,Centre of Excellence in Maritime Safety,Singapore"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Singapore Polytechnic,Centre of Excellence in Maritime Safety,Singapore","institution_ids":["https://openalex.org/I202502355"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5054070832","display_name":"Yisi Liu","orcid":"https://orcid.org/0000-0001-6974-1462"},"institutions":[{"id":"https://openalex.org/I202502355","display_name":"Singapore Polytechnic","ror":"https://ror.org/05pxcn248","country_code":"SG","type":"education","lineage":["https://openalex.org/I202502355"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Yisi Liu","raw_affiliation_strings":["Singapore Polytechnic,Centre of Excellence in Maritime Safety,Singapore"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Singapore Polytechnic,Centre of Excellence in Maritime Safety,Singapore","institution_ids":["https://openalex.org/I202502355"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I202502355"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.40068571,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"358","last_page":"361"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10525","display_name":"Human-Automation Interaction and Safety","score":0.18719999492168427,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10525","display_name":"Human-Automation Interaction and Safety","score":0.18719999492168427,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11622","display_name":"Maritime Navigation and Safety","score":0.1868000030517578,"subfield":{"id":"https://openalex.org/subfields/2212","display_name":"Ocean Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.04749999940395355,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/checklist","display_name":"Checklist","score":0.8241999745368958},{"id":"https://openalex.org/keywords/protocol","display_name":"Protocol (science)","score":0.6108999848365784},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5425999760627747},{"id":"https://openalex.org/keywords/compliance","display_name":"Compliance (psychology)","score":0.5414999723434448},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.49410000443458557},{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.48100000619888306}],"concepts":[{"id":"https://openalex.org/C2779356329","wikidata":"https://www.wikidata.org/wiki/Q922625","display_name":"Checklist","level":2,"score":0.8241999745368958},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6827999949455261},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.6108999848365784},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5425999760627747},{"id":"https://openalex.org/C2781460075","wikidata":"https://www.wikidata.org/wiki/Q1399332","display_name":"Compliance (psychology)","level":2,"score":0.5414999723434448},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.49410000443458557},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4812999963760376},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.48100000619888306},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4796999990940094},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4133000075817108},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.37950000166893005},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.36469998955726624},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.33899998664855957},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.2745000123977661},{"id":"https://openalex.org/C12269588","wikidata":"https://www.wikidata.org/wiki/Q132364","display_name":"Communications protocol","level":2,"score":0.2689000070095062},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2574999928474426},{"id":"https://openalex.org/C122783720","wikidata":"https://www.wikidata.org/wiki/Q183065","display_name":"Interpreter","level":2,"score":0.25609999895095825}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/cw68232.2025.00059","is_oa":false,"landing_page_url":"https://doi.org/10.1109/cw68232.2025.00059","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 International Conference on Cyberworlds (CW\uff09","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2508.08652","is_oa":true,"landing_page_url":"https://arxiv.org/abs/2508.08652","pdf_url":"https://arxiv.org/pdf/2508.08652","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2508.08652","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2508.08652","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2508.08652","is_oa":true,"landing_page_url":"https://arxiv.org/abs/2508.08652","pdf_url":"https://arxiv.org/pdf/2508.08652","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4416863255.pdf","grobid_xml":"https://content.openalex.org/works/W4416863255.grobid-xml"},"referenced_works_count":3,"referenced_works":["https://openalex.org/W4406152279","https://openalex.org/W4386332795","https://openalex.org/W4405633694"],"related_works":[],"abstract_inverted_index":{"Accurate":[0],"evaluation":[1],"of":[2,163],"procedural":[3],"communication":[4],"compliance":[5,18,129],"is":[6,121],"essential":[7],"in":[8,12,61,78,165,173],"simulation-based":[9],"training,":[10],"particularly":[11],"safety-critical":[13],"domains":[14],"where":[15],"adherence":[16],"to":[17,55],"checklists":[19],"reflects":[20],"operational":[21],"competence.":[22],"This":[23,157],"paper":[24],"explores":[25],"a":[26,49,62,75,115,128],"lightweight,":[27],"deployable":[28],"approach":[29],"using":[30,139],"prompt-based":[31],"inference":[32],"with":[33,82,91],"open-source":[34],"large":[35],"language":[36],"models":[37,92],"(LLMs)":[38],"that":[39,51,148],"can":[40],"run":[41],"efficiently":[42],"on":[43,69,106],"consumer-grade":[44],"GPUs.":[45],"We":[46,73,131],"present":[47],"Prompt-and-Check,":[48],"method":[50],"uses":[52],"context-rich":[53],"prompts":[54],"evaluate":[56],"whether":[57],"each":[58,112],"checklist":[59,113],"item":[60],"protocol":[63],"has":[64],"been":[65],"fulfilled,":[66],"solely":[67],"based":[68],"transcribed":[70],"verbal":[71],"exchanges.":[72],"perform":[74],"case":[76],"study":[77,158],"the":[79,124,160],"maritime":[80],"domain":[81],"participants":[83],"performing":[84],"an":[85,107],"identical":[86],"simulation":[87],"task,":[88],"and":[89,101,142,170],"experiment":[90],"such":[93],"as":[94],"LLama":[95],"2":[96],"7B,":[97,103],"LLaMA":[98],"3":[99],"8B":[100],"Mistral":[102],"running":[104],"locally":[105],"RTX":[108],"4070":[109],"GPU.":[110],"For":[111],"item,":[114],"prompt":[116],"incorporating":[117],"relevant":[118],"transcript":[119],"excerpts":[120],"fed":[122],"into":[123],"model,":[125],"which":[126],"outputs":[127,134],"judgment.":[130],"assess":[132],"model":[133],"against":[135],"expert-annotated":[136],"ground":[137],"truth":[138],"classification":[140],"accuracy":[141],"agreement":[143],"scores.":[144],"Our":[145],"findings":[146],"demonstrate":[147],"prompting":[149],"enables":[150],"effective":[151],"context-aware":[152],"reasoning":[153],"without":[154],"task-specific":[155],"training.":[156],"highlights":[159],"practical":[161],"utility":[162],"LLMs":[164],"augmenting":[166],"debriefing,":[167],"performance":[168],"feedback,":[169],"automated":[171],"assessment":[172],"training":[174],"environments.":[175]},"counts_by_year":[],"updated_date":"2026-08-26T07:47:46.906454","created_date":"2025-10-10T00:00:00"}