{"id":"https://openalex.org/W7204806104","doi":"https://doi.org/10.64898/2026.08.26.26361437","title":"Agreement between an artificial intelligence video-analysis platform and human evaluators scoring emergency medical technician psychomotor skills: a fully crossed reliability study","display_name":"Agreement between an artificial intelligence video-analysis platform and human evaluators scoring emergency medical technician psychomotor skills: a fully crossed reliability study","publication_year":2026,"publication_date":"2026-08-31","ids":{"openalex":"https://openalex.org/W7204806104","doi":"https://doi.org/10.64898/2026.08.26.26361437"},"language":"en","primary_location":{"id":"doi:10.64898/2026.08.26.26361437","doi":"https://doi.org/10.64898/2026.08.26.26361437","is_oa":true,"landing_page_url":"https://doi.org/10.64898/2026.08.26.26361437","pdf_url":null,"source":{"id":"https://openalex.org/S3005729997","display_name":"medRxiv","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":["https://openalex.org/P4310315909"],"host_organization_lineage_names":["Cold Spring Harbor Laboratory Press"],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"medRxiv","raw_type":"posted-content"},"volume":null,"type":"preprint","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.64898/2026.08.26.26361437","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5148759476","display_name":"Joel H Otte","orcid":"https://orcid.org/0009-0003-1616-0076","observed_orcids":["https://orcid.org/0009-0003-1616-0076"]},"institutions":[{"id":"https://openalex.org/I130915974","display_name":"Des Moines Area Community College","ror":"https://ror.org/0016w2w54","country_code":"US","type":"education","lineage":["https://openalex.org/I130915974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Joel H. Otte","raw_affiliation_strings":["Emergency Medical Services and Fire Science Education, Des Moines Area Community College, West Des Moines, Iowa, USA"],"raw_orcid":"https://orcid.org/0009-0003-1616-0076","affiliations":[{"raw_affiliation_string":"Emergency Medical Services and Fire Science Education, Des Moines Area Community College, West Des Moines, Iowa, USA","institution_ids":["https://openalex.org/I130915974"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5148796602","display_name":"Autumn Cartagena","orcid":"https://orcid.org/0009-0004-3675-7934","observed_orcids":["https://orcid.org/0009-0004-3675-7934"]},"institutions":[{"id":"https://openalex.org/I130915974","display_name":"Des Moines Area Community College","ror":"https://ror.org/0016w2w54","country_code":"US","type":"education","lineage":["https://openalex.org/I130915974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Autumn Cartagena","raw_affiliation_strings":["Institutional Effectiveness Department, Des Moines Area Community College, Ankeny, Iowa, USA"],"raw_orcid":"https://orcid.org/0009-0004-3675-7934","affiliations":[{"raw_affiliation_string":"Institutional Effectiveness Department, Des Moines Area Community College, Ankeny, Iowa, USA","institution_ids":["https://openalex.org/I130915974"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I130915974"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10254","display_name":"Innovations in Medical Education","score":0.6410999894142151,"subfield":{"id":"https://openalex.org/subfields/2739","display_name":"Public Health, Environmental and Occupational Health"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T10254","display_name":"Innovations in Medical Education","score":0.6410999894142151,"subfield":{"id":"https://openalex.org/subfields/2739","display_name":"Public Health, Environmental and Occupational Health"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T11238","display_name":"Simulation-Based Education in Healthcare","score":0.05290000140666962,"subfield":{"id":"https://openalex.org/subfields/2737","display_name":"Physiology (medical)"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T14354","display_name":"Emergency Medicine Education and Research","score":0.043800000101327896,"subfield":{"id":"https://openalex.org/subfields/2711","display_name":"Emergency Medicine"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/emergency-medical-services-education","display_name":"emergency medical services education","score":0.8980000019073486},{"id":"https://openalex.org/keywords/inter-rater-reliability","display_name":"inter-rater reliability","score":0.847000002861023},{"id":"https://openalex.org/keywords/bag-mask-ventilation","display_name":"bag-mask ventilation","score":0.9980000257492065},{"id":"https://openalex.org/keywords/medical-evaluation","display_name":"medical assessment","score":0.7739999890327454},{"id":"https://openalex.org/keywords/critical-failures","display_name":"critical failures","score":0.6190000176429749},{"id":"https://openalex.org/keywords/automated-assessment","display_name":"automated assessment","score":0.3720000088214874}],"concepts":[{"id":"https://openalex.org/C191147762","wikidata":"https://www.wikidata.org/wiki/Q186289","display_name":"Human reliability","level":3,"score":0.6252999901771545},{"id":"https://openalex.org/C42525527","wikidata":"https://www.wikidata.org/wiki/Q1209955","display_name":"Formative assessment","level":2,"score":0.5726000070571899},{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.5533000230789185},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5328999757766724},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.513700008392334},{"id":"https://openalex.org/C2779356329","wikidata":"https://www.wikidata.org/wiki/Q922625","display_name":"Checklist","level":2,"score":0.5102999806404114},{"id":"https://openalex.org/C164953862","wikidata":"https://www.wikidata.org/wiki/Q7256440","display_name":"Psychomotor learning","level":3,"score":0.43479999899864197},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4228000044822693},{"id":"https://openalex.org/C75630572","wikidata":"https://www.wikidata.org/wiki/Q538904","display_name":"Applied psychology","level":1,"score":0.3612000048160553},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.35910001397132874},{"id":"https://openalex.org/C61863361","wikidata":"https://www.wikidata.org/wiki/Q470749","display_name":"Inter-rater reliability","level":3,"score":0.3580000102519989},{"id":"https://openalex.org/C46304622","wikidata":"https://www.wikidata.org/wiki/Q374814","display_name":"Certification","level":2,"score":0.33709999918937683},{"id":"https://openalex.org/C2777230681","wikidata":"https://www.wikidata.org/wiki/Q7923820","display_name":"Vetting","level":2,"score":0.3352999985218048},{"id":"https://openalex.org/C2776818064","wikidata":"https://www.wikidata.org/wiki/Q829903","display_name":"Agreement","level":2,"score":0.3046000003814697},{"id":"https://openalex.org/C2780310539","wikidata":"https://www.wikidata.org/wiki/Q12547192","display_name":"Imperfect","level":2,"score":0.2797999978065491},{"id":"https://openalex.org/C44154836","wikidata":"https://www.wikidata.org/wiki/Q45045","display_name":"Simulation","level":1,"score":0.2784000039100647},{"id":"https://openalex.org/C523889960","wikidata":"https://www.wikidata.org/wiki/Q160695","display_name":"Torso","level":2,"score":0.2754000127315521},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2700999975204468},{"id":"https://openalex.org/C145804949","wikidata":"https://www.wikidata.org/wiki/Q478123","display_name":"Situation awareness","level":2,"score":0.2653000056743622},{"id":"https://openalex.org/C5457282","wikidata":"https://www.wikidata.org/wiki/Q7907352","display_name":"VNIR","level":3,"score":0.26249998807907104},{"id":"https://openalex.org/C509550671","wikidata":"https://www.wikidata.org/wiki/Q126945","display_name":"Medical education","level":1,"score":0.259799987077713},{"id":"https://openalex.org/C107418235","wikidata":"https://www.wikidata.org/wiki/Q1520565","display_name":"Human multitasking","level":2,"score":0.25029999017715454}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.64898/2026.08.26.26361437","doi":"https://doi.org/10.64898/2026.08.26.26361437","is_oa":true,"landing_page_url":"https://doi.org/10.64898/2026.08.26.26361437","pdf_url":null,"source":{"id":"https://openalex.org/S3005729997","display_name":"medRxiv","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":["https://openalex.org/P4310315909"],"host_organization_lineage_names":["Cold Spring Harbor Laboratory Press"],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"medRxiv","raw_type":"posted-content"}],"best_oa_location":{"id":"doi:10.64898/2026.08.26.26361437","doi":"https://doi.org/10.64898/2026.08.26.26361437","is_oa":true,"landing_page_url":"https://doi.org/10.64898/2026.08.26.26361437","pdf_url":null,"source":{"id":"https://openalex.org/S3005729997","display_name":"medRxiv","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":["https://openalex.org/P4310315909"],"host_organization_lineage_names":["Cold Spring Harbor Laboratory Press"],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"medRxiv","raw_type":"posted-content"},"sustainable_development_goals":[{"score":0.7246000170707703,"id":"https://metadata.un.org/sdg/3","display_name":"Good health and well-being"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":20,"referenced_works":["https://openalex.org/W1751360006","https://openalex.org/W1975879668","https://openalex.org/W1996290308","https://openalex.org/W2141403362","https://openalex.org/W2164777277","https://openalex.org/W2327037637","https://openalex.org/W2608645592","https://openalex.org/W3021916629","https://openalex.org/W3106889297","https://openalex.org/W4214584911","https://openalex.org/W4385850949","https://openalex.org/W4402366310","https://openalex.org/W4406325012","https://openalex.org/W4406756892","https://openalex.org/W4412408134","https://openalex.org/W7106271742","https://openalex.org/W7118180601","https://openalex.org/W7124709896","https://openalex.org/W7168022660","https://openalex.org/W7171631545"],"related_works":["https://openalex.org/W1981637016","https://openalex.org/W7168427803","https://openalex.org/W7208836352","https://openalex.org/W7219714825","https://openalex.org/W3216248330","https://openalex.org/W1007071250","https://openalex.org/W7163813762","https://openalex.org/W2045019287","https://openalex.org/W1986481813","https://openalex.org/W4409989142"],"abstract_inverted_index":{"Abstract":[0],"Objectives":[1],"Psychomotor":[2],"competency":[3],"verification":[4],"for":[5,124,129,134,168,200,223,231,235,244,250],"emergency":[6],"medical":[7,98,205,236],"technician":[8],"(EMT)":[9],"candidates":[10],"is":[11,291,296],"set":[12],"by":[13],"each":[14],"state":[15],"and,":[16],"where":[17,271,300],"states":[18],"approve":[19],"program-verified":[20],"competency,":[21],"rests":[22],"on":[23,47,153,173,184,194,274],"programs":[24],"and":[25,73,97,116,127,131,139,193,204,233,247,269,282],"their":[26],"local":[27],"evaluators.":[28],"Commercial":[29],"artificial":[30],"intelligence":[31],"(AI)":[32],"video-analysis":[33],"platforms":[34],"are":[35,285],"marketed":[36],"to":[37],"programs,":[38],"but":[39,182],"none":[40],"has":[41],"been":[42],"evaluated":[43,63],"against":[44,66,166],"human":[45,71,136,228,294],"evaluators":[46,57,72],"EMT":[48],"skills.":[49],"We":[50,109],"measured":[51],"the":[52,135,141,169,185,195,218,239,251,265],"inter-rater":[53],"reliability":[54,161,243],"of":[55,81],"certified":[56],"scoring":[58],"identical":[59,103],"video-recorded":[60],"performances,":[61],"then":[62],"one":[64,74],"platform":[65,76,142,240,255],"that":[67],"baseline.":[68],"Methods":[69],"Five":[70],"AI":[75],"independently":[77],"scored":[78],"every":[79,227],"video":[80],"three":[82,155],"skills":[83,156],"in":[84],"a":[85,144,257],"fully":[86],"crossed":[87],"design:":[88],"cervical":[89,201,224,245],"collar":[90,202,246],"application":[91],"(15":[92],"performances),":[93],"bag-valve-mask":[94],"ventilation":[95],"(14),":[96],"patient":[99],"assessment":[100,206],"(15),":[101],"using":[102],"dichotomous":[104],"checklists":[105],"with":[106,140,217],"critical-failure":[107,130,196],"criteria.":[108],"computed":[110],"item-level":[111,149],"Fleiss":[112],"\u03ba,":[113],"Krippendorff":[114],"\u03b1,":[115],"Gwet":[117],"AC1;":[118],"absolute-agreement":[119],"intraclass":[120],"correlation":[121,216],"coefficients":[122],"(ICC)":[123],"percentage":[125],"scores;":[126],"\u03ba":[128],"pass/fail":[132,187],"decisions,":[133],"panel":[137,242,266],"alone":[138],"as":[143,262,264],"sixth":[145],"rater.":[146],"Results":[147],"Human":[148],"agreement":[150,183],"was":[151,162,178,189,221],"moderate":[152,163,179],"all":[154],"(\u03ba":[157,180,191],"0.41\u20130.47).":[158],"Single-rater":[159],"score":[160],"(ICC":[164],"0.54\u20130.69)":[165],"0.85\u20130.92":[167],"five-rater":[170],"mean.":[171],"Agreement":[172],"an":[174],"objective":[175],"75%":[176],"threshold":[177],"0.54\u20130.57),":[181],"recorded":[186],"decision":[188],"fair":[190],"0.30\u20130.39)":[192],"determination":[197],"near":[198],"zero":[199],"(0.03)":[203],"(0.12).":[207],"Platform":[208],"alignment":[209],"tracked":[210],"rubric":[211],"observability,":[212],"not":[213],"task":[214],"complexity:":[215],"five-human":[219],"mean":[220],"0.857":[222],"collar,":[225],"exceeding":[226],"evaluator,":[229],"\u22120.173":[230],"bag-valve-mask,":[232],"0.664":[234],"assessment.":[237],"Adding":[238],"improved":[241],"reduced":[248],"it":[249,267,301],"others.":[252],"Conclusions":[253],"The":[254,293],"rated":[256],"discrete,":[258],"visually":[259],"observable":[260],"skill":[261],"well":[263],"joined":[268],"degraded":[270],"credit":[272],"depended":[273],"rates,":[275],"volumes,":[276],"or":[277],"verbalization.":[278],"Formative":[279],"practice":[280],"feedback":[281],"adjunct":[283],"rating":[284],"defensible":[286],"now;":[287],"autonomous":[288],"summative":[289],"use":[290],"not.":[292],"baseline":[295],"least":[297],"reliable":[298],"precisely":[299],"decides":[302],"outcomes.":[303]},"counts_by_year":[],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2026-09-01T00:00:00"}