{"id":"https://openalex.org/W7204695968","doi":"https://doi.org/10.62411/jcta.16829","title":"Transformer-Based Support for Content-Validity Pre-Screening in Educational Materials","display_name":"Transformer-Based Support for Content-Validity Pre-Screening in Educational Materials","publication_year":2026,"publication_date":"2026-08-31","ids":{"openalex":"https://openalex.org/W7204695968","doi":"https://doi.org/10.62411/jcta.16829"},"language":"en","primary_location":{"id":"doi:10.62411/jcta.16829","doi":"https://doi.org/10.62411/jcta.16829","is_oa":true,"landing_page_url":"https://doi.org/10.62411/jcta.16829","pdf_url":"https://publikasi.dinus.ac.id/jcta/article/download/16829/6285","source":{"id":"https://openalex.org/S4387292246","display_name":"Journal of Computing Theories and Applications","issn_l":"3024-9104","issn":["3024-9104"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310315221","host_organization_name":"Nuswantoro Dian University","host_organization_lineage":["https://openalex.org/P4310315221"],"host_organization_lineage_names":["Nuswantoro Dian University"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Computing Theories and Applications","raw_type":"journal-article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://publikasi.dinus.ac.id/jcta/article/download/16829/6285","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5018799402","display_name":"Safuan Safuan","orcid":"https://orcid.org/0000-0002-6680-8059","observed_orcids":["https://orcid.org/0000-0002-6680-8059"]},"institutions":[{"id":"https://openalex.org/I4210159899","display_name":"Universitas Muhammadiyah Semarang","ror":"https://ror.org/05hra0856","country_code":"ID","type":"education","lineage":["https://openalex.org/I4210159899"]}],"countries":["ID"],"is_corresponding":false,"raw_author_name":"Safuan Safuan","raw_affiliation_strings":["Universitas Muhammadiyah Semarang"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Universitas Muhammadiyah Semarang","institution_ids":["https://openalex.org/I4210159899"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066093413","display_name":"Dhendra Marutho","orcid":"https://orcid.org/0009-0004-2271-1767","observed_orcids":["https://orcid.org/0009-0004-2271-1767"]},"institutions":[{"id":"https://openalex.org/I4210159899","display_name":"Universitas Muhammadiyah Semarang","ror":"https://ror.org/05hra0856","country_code":"ID","type":"education","lineage":["https://openalex.org/I4210159899"]}],"countries":["ID"],"is_corresponding":false,"raw_author_name":"Dhendra Marutho","raw_affiliation_strings":["Universitas Muhammadiyah Semarang"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Universitas Muhammadiyah Semarang","institution_ids":["https://openalex.org/I4210159899"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090944980","display_name":"Ahmad Ilham","orcid":"https://orcid.org/0000-0001-5109-6258","observed_orcids":["https://orcid.org/0000-0001-5109-6258"]},"institutions":[{"id":"https://openalex.org/I4210159899","display_name":"Universitas Muhammadiyah Semarang","ror":"https://ror.org/05hra0856","country_code":"ID","type":"education","lineage":["https://openalex.org/I4210159899"]}],"countries":["ID"],"is_corresponding":false,"raw_author_name":"Ahmad Ilham","raw_affiliation_strings":["Universitas Muhammadiyah Semarang"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Universitas Muhammadiyah Semarang","institution_ids":["https://openalex.org/I4210159899"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008884120","display_name":"Muhammad Munsarif","orcid":"https://orcid.org/0000-0002-3009-4234","observed_orcids":["https://orcid.org/0000-0002-3009-4234"]},"institutions":[{"id":"https://openalex.org/I4210159899","display_name":"Universitas Muhammadiyah Semarang","ror":"https://ror.org/05hra0856","country_code":"ID","type":"education","lineage":["https://openalex.org/I4210159899"]}],"countries":["ID"],"is_corresponding":false,"raw_author_name":"Muhammad Munsarif","raw_affiliation_strings":["Universitas Muhammadiyah Semarang"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Universitas Muhammadiyah Semarang","institution_ids":["https://openalex.org/I4210159899"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016969382","display_name":"Wendy Sarasjati","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210159899","display_name":"Universitas Muhammadiyah Semarang","ror":"https://ror.org/05hra0856","country_code":"ID","type":"education","lineage":["https://openalex.org/I4210159899"]}],"countries":["ID"],"is_corresponding":false,"raw_author_name":"Wendy Sarasjati","raw_affiliation_strings":["Universitas Muhammadiyah Semarang"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Universitas Muhammadiyah Semarang","institution_ids":["https://openalex.org/I4210159899"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050010244","display_name":"Edy Winarno","orcid":"https://orcid.org/0000-0002-7488-5660","observed_orcids":["https://orcid.org/0000-0002-7488-5660"]},"institutions":[{"id":"https://openalex.org/I4210159899","display_name":"Universitas Muhammadiyah Semarang","ror":"https://ror.org/05hra0856","country_code":"ID","type":"education","lineage":["https://openalex.org/I4210159899"]}],"countries":["ID"],"is_corresponding":false,"raw_author_name":"Edy Winarno","raw_affiliation_strings":["Universitas Muhammadiyah Semarang"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Universitas Muhammadiyah Semarang","institution_ids":["https://openalex.org/I4210159899"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049676192","display_name":"Arnold Adimabua Ojugo","orcid":"https://orcid.org/0000-0003-4150-5163","observed_orcids":["https://orcid.org/0000-0003-4150-5163"]},"institutions":[{"id":"https://openalex.org/I891860020","display_name":"Federal University of Petroleum Resource Effurun","ror":"https://ror.org/04ndqkb04","country_code":"NG","type":"education","lineage":["https://openalex.org/I891860020"]}],"countries":["NG"],"is_corresponding":false,"raw_author_name":"Arnold Adimabua Ojugo","raw_affiliation_strings":["Federal University of Petroleum Resources"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Federal University of Petroleum Resources","institution_ids":["https://openalex.org/I891860020"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5019714454","display_name":"De Rosal Ignatius Moses Setiadi","orcid":"https://orcid.org/0000-0001-6615-4457","observed_orcids":["https://orcid.org/0000-0001-6615-4457"]},"institutions":[{"id":"https://openalex.org/I4210127958","display_name":"Universitas Dian Nuswantoro","ror":"https://ror.org/02csxcg02","country_code":"ID","type":"education","lineage":["https://openalex.org/I4210127958"]}],"countries":["ID"],"is_corresponding":false,"raw_author_name":"De Rosal Ignatius Moses Setiadi","raw_affiliation_strings":["Universitas Dian Nuswantoro"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Universitas Dian Nuswantoro","institution_ids":["https://openalex.org/I4210127958"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.3951,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.80881696,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":"4","issue":"1","first_page":"423","last_page":"442"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6431000232696533,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6431000232696533,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13117","display_name":"Health Education and Validation","score":0.06759999692440033,"subfield":{"id":"https://openalex.org/subfields/2922","display_name":"Research and Theory"},"field":{"id":"https://openalex.org/fields/29","display_name":"Nursing"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.04879999905824661,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/content-validity","display_name":"content validity","score":0.9539999961853027},{"id":"https://openalex.org/keywords/teaching-materials","display_name":"teaching materials","score":0.7379999756813049},{"id":"https://openalex.org/keywords/aiken-s-v","display_name":"Aiken's V","score":0.7590000033378601},{"id":"https://openalex.org/keywords/transformer","display_name":"transformer models","score":0.6899999976158142},{"id":"https://openalex.org/keywords/multilingual-bert","display_name":"multilingual BERT","score":0.9380000233650208},{"id":"https://openalex.org/keywords/deberta","display_name":"DeBERTa","score":0.6859999895095825},{"id":"https://openalex.org/keywords/xlm-r","display_name":"XLM-R","score":0.8519999980926514},{"id":"https://openalex.org/keywords/indobert","display_name":"IndoBERT","score":0.7910000085830688},{"id":"https://openalex.org/keywords/expert-opinion","display_name":"expert opinion","score":0.6679999828338623},{"id":"https://openalex.org/keywords/human-in-the-loop-decision-support","display_name":"human-in-the-loop decision support","score":0.8700000047683716},{"id":"https://openalex.org/keywords/learning-outcomes","display_name":"learning outcomes","score":0.4659999907016754},{"id":"https://openalex.org/keywords/information-leakage","display_name":"information leakage","score":0.37299999594688416}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5450000166893005},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.41290000081062317},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3864000141620636},{"id":"https://openalex.org/C117220453","wikidata":"https://www.wikidata.org/wiki/Q5172842","display_name":"Correlation","level":2,"score":0.3610999882221222},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.3427000045776367},{"id":"https://openalex.org/C188231857","wikidata":"https://www.wikidata.org/wiki/Q2585724","display_name":"Educational assessment","level":2,"score":0.31779998540878296},{"id":"https://openalex.org/C100521375","wikidata":"https://www.wikidata.org/wiki/Q2015382","display_name":"Competence (human resources)","level":2,"score":0.3111000061035156},{"id":"https://openalex.org/C162446236","wikidata":"https://www.wikidata.org/wiki/Q653137","display_name":"Content analysis","level":2,"score":0.2888999879360199},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2784000039100647},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.27469998598098755},{"id":"https://openalex.org/C3018824978","wikidata":"https://www.wikidata.org/wiki/Q2894891","display_name":"Error analysis","level":2,"score":0.2694999873638153},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.266400009393692},{"id":"https://openalex.org/C2992052996","wikidata":"https://www.wikidata.org/wiki/Q27318","display_name":"Educational testing","level":3,"score":0.2590999901294708}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.62411/jcta.16829","doi":"https://doi.org/10.62411/jcta.16829","is_oa":true,"landing_page_url":"https://doi.org/10.62411/jcta.16829","pdf_url":"https://publikasi.dinus.ac.id/jcta/article/download/16829/6285","source":{"id":"https://openalex.org/S4387292246","display_name":"Journal of Computing Theories and Applications","issn_l":"3024-9104","issn":["3024-9104"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310315221","host_organization_name":"Nuswantoro Dian University","host_organization_lineage":["https://openalex.org/P4310315221"],"host_organization_lineage_names":["Nuswantoro Dian University"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Computing Theories and Applications","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.62411/jcta.16829","doi":"https://doi.org/10.62411/jcta.16829","is_oa":true,"landing_page_url":"https://doi.org/10.62411/jcta.16829","pdf_url":"https://publikasi.dinus.ac.id/jcta/article/download/16829/6285","source":{"id":"https://openalex.org/S4387292246","display_name":"Journal of Computing Theories and Applications","issn_l":"3024-9104","issn":["3024-9104"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310315221","host_organization_name":"Nuswantoro Dian University","host_organization_lineage":["https://openalex.org/P4310315221"],"host_organization_lineage_names":["Nuswantoro Dian University"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Computing Theories and Applications","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.975600004196167,"display_name":"Quality education","id":"https://metadata.un.org/sdg/4"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7204695968.pdf","grobid_xml":"https://content.openalex.org/works/W7204695968.grobid-xml"},"referenced_works_count":27,"referenced_works":["https://openalex.org/W2107224451","https://openalex.org/W2146456356","https://openalex.org/W2172059128","https://openalex.org/W2883295433","https://openalex.org/W2905810301","https://openalex.org/W2954017678","https://openalex.org/W2962873892","https://openalex.org/W2963341956","https://openalex.org/W2983040767","https://openalex.org/W3092129462","https://openalex.org/W3103252638","https://openalex.org/W3157252883","https://openalex.org/W4293589102","https://openalex.org/W4366563382","https://openalex.org/W4376646476","https://openalex.org/W4391070402","https://openalex.org/W4401393088","https://openalex.org/W4401702825","https://openalex.org/W4402056890","https://openalex.org/W4404518574","https://openalex.org/W4405351517","https://openalex.org/W4406805313","https://openalex.org/W4408311134","https://openalex.org/W4410831815","https://openalex.org/W4411613691","https://openalex.org/W4417180657","https://openalex.org/W7202257054"],"related_works":["https://openalex.org/W4312659575","https://openalex.org/W7168408336","https://openalex.org/W4415286521","https://openalex.org/W7167425501","https://openalex.org/W7155158004","https://openalex.org/W4323520585","https://openalex.org/W7171524391","https://openalex.org/W7217575955","https://openalex.org/W7165391004","https://openalex.org/W4413010199"],"abstract_inverted_index":{"Content":[0],"validity":[1],"assessment":[2,17],"is":[3],"essential":[4],"for":[5,205,227],"determining":[6],"whether":[7],"educational":[8,61,211],"materials":[9],"adequately":[10],"represent":[11],"intended":[12],"learning":[13],"outcomes.":[14],"However,":[15,213],"conventional":[16],"procedures":[18],"require":[19],"substantial":[20],"expert":[21,228],"time":[22],"and":[23,51,75,97,117,143,157,230,235],"may":[24],"produce":[25],"inconsistent":[26],"decisions":[27],"across":[28,233],"large":[29],"item":[30],"collections.":[31],"This":[32],"study":[33],"develops":[34],"a":[35,82,220,225],"transformer-based":[36],"framework":[37,215],"to":[38],"support":[39,195],"content-validity":[40],"pre-screening":[41,221],"through":[42],"two":[43],"complementary":[44],"tasks:":[45],"predicting":[46],"expert-derived":[47,172],"Aiken\u2019s":[48,101],"V":[49,102],"coefficients":[50],"classifying":[52],"instructional-item":[53],"essentiality.":[54],"The":[55,193],"final":[56],"dataset":[57],"comprised":[58],"652":[59],"Indonesian-language":[60],"text":[62],"items":[63],"independently":[64],"evaluated":[65],"by":[66],"four":[67],"subject-matter":[68],"experts.":[69],"To":[70],"reduce":[71],"information":[72],"leakage,":[73],"identical":[74],"normalized-equivalent":[76],"texts":[77],"were":[78,90],"grouped":[79],"before":[80],"applying":[81],"group-aware":[83],"70:15:15":[84],"training\u2013validation\u2013test":[85],"split.":[86],"Classical":[87],"TF-IDF-based":[88],"baselines":[89],"compared":[91],"with":[92,171],"IndoBERT,":[93],"multilingual":[94,98,104,124,148],"BERT,":[95],"XLM-RoBERTa,":[96],"DeBERTa-v3.":[99],"For":[100,133],"regression,":[103],"BERT":[105,149],"achieved":[106,126,137,150],"the":[107,112,118,127,138,151,165,196,214],"lowest":[108,113],"MAE":[109],"of":[110,115,121,131,141,145,155,159,198],"0.0501,":[111],"RMSE":[114],"0.0625,":[116],"highest":[119,128,139,152],"R\u00b2":[120],"0.5239,":[122],"whereas":[123,147],"DeBERTa-v3":[125],"Spearman":[129],"correlation":[130],"0.7532.":[132],"essentiality":[134],"classification,":[135],"XLM-RoBERTa":[136],"accuracy":[140,154],"0.8557":[142],"Macro-F1":[144],"0.8161,":[146],"balanced":[153],"0.8135":[156],"ROC-AUC":[158],"0.9111.":[160],"Error":[161],"analysis":[162],"showed":[163],"that":[164],"models":[166,200],"captured":[167],"textual":[168],"patterns":[169],"associated":[170],"outcomes":[173],"but":[174],"remained":[175],"limited":[176],"when":[177],"judgments":[178],"depended":[179],"on":[180],"broader":[181],"curricular":[182],"context,":[183],"competency":[184],"hierarchies,":[185],"prerequisite":[186],"relationships,":[187],"or":[188,208],"relationships":[189],"among":[190],"instructional":[191],"items.":[192,212],"findings":[194],"use":[197],"transformer":[199],"as":[201,219],"human-in-the-loop":[202],"decision-support":[203],"tools":[204],"prioritizing":[206],"uncertain":[207],"potentially":[209],"problematic":[210],"should":[216],"be":[217],"interpreted":[218],"mechanism":[222],"rather":[223],"than":[224],"replacement":[226],"judgment,":[229],"external":[231],"validation":[232],"institutions":[234],"disciplines":[236],"remains":[237],"necessary.":[238]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2026-08-31T00:00:00"}