{"id":"https://openalex.org/W4408320960","doi":"https://doi.org/10.1111/bjet.13570","title":"Leveraging <scp>LLM</scp> respondents for item evaluation: A psychometric analysis","display_name":"Leveraging <scp>LLM</scp> respondents for item evaluation: A psychometric analysis","publication_year":2025,"publication_date":"2025-02-24","ids":{"openalex":"https://openalex.org/W4408320960","doi":"https://doi.org/10.1111/bjet.13570"},"language":"en","primary_location":{"id":"doi:10.1111/bjet.13570","is_oa":true,"landing_page_url":"https://doi.org/10.1111/bjet.13570","pdf_url":"https://onlinelibrary.wiley.com/doi/pdfdirect/10.1111/bjet.13570","source":{"id":"https://openalex.org/S110346167","display_name":"British Journal of Educational Technology","issn_l":"0007-1013","issn":["0007-1013","1467-8535"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"British Journal of Educational Technology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://onlinelibrary.wiley.com/doi/pdfdirect/10.1111/bjet.13570","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5111259533","display_name":"Yunting Liu","orcid":null},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yunting Liu","raw_affiliation_strings":["School of Education University of California, Berkeley  Berkeley California USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Education University of California, Berkeley  Berkeley California USA","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101859450","display_name":"Shreya Bhandari","orcid":"https://orcid.org/0009-0007-1705-3052"},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shreya Bhandari","raw_affiliation_strings":["School of Education University of California, Berkeley  Berkeley California USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Education University of California, Berkeley  Berkeley California USA","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5021273980","display_name":"Zachary A. Pardos","orcid":"https://orcid.org/0000-0002-6016-7051"},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Zachary A. Pardos","raw_affiliation_strings":["School of Education University of California, Berkeley  Berkeley California USA"],"raw_orcid":"https://orcid.org/0000-0002-6016-7051","affiliations":[{"raw_affiliation_string":"School of Education University of California, Berkeley  Berkeley California USA","institution_ids":["https://openalex.org/I95457486"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5021273980"],"corresponding_institution_ids":["https://openalex.org/I95457486"],"apc_list":{"value":3860,"currency":"USD","value_usd":3860},"apc_paid":{"value":3860,"currency":"USD","value_usd":3860},"fwci":42.0306,"has_fulltext":true,"cited_by_count":25,"citation_normalized_percentile":{"value":0.99769218,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"56","issue":"3","first_page":"1028","last_page":"1052"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.982699990272522,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.982699990272522,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.9546999931335449,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9502000212669373,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.5882819890975952},{"id":"https://openalex.org/keywords/item-analysis","display_name":"Item analysis","score":0.5109924077987671},{"id":"https://openalex.org/keywords/psychometrics","display_name":"Psychometrics","score":0.44013601541519165},{"id":"https://openalex.org/keywords/applied-psychology","display_name":"Applied psychology","score":0.4086894690990448},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.35984688997268677},{"id":"https://openalex.org/keywords/clinical-psychology","display_name":"Clinical psychology","score":0.2615482211112976}],"concepts":[{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.5882819890975952},{"id":"https://openalex.org/C2778263599","wikidata":"https://www.wikidata.org/wiki/Q1669245","display_name":"Item analysis","level":3,"score":0.5109924077987671},{"id":"https://openalex.org/C171606756","wikidata":"https://www.wikidata.org/wiki/Q506132","display_name":"Psychometrics","level":2,"score":0.44013601541519165},{"id":"https://openalex.org/C75630572","wikidata":"https://www.wikidata.org/wiki/Q538904","display_name":"Applied psychology","level":1,"score":0.4086894690990448},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.35984688997268677},{"id":"https://openalex.org/C70410870","wikidata":"https://www.wikidata.org/wiki/Q199906","display_name":"Clinical psychology","level":1,"score":0.2615482211112976}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1111/bjet.13570","is_oa":true,"landing_page_url":"https://doi.org/10.1111/bjet.13570","pdf_url":"https://onlinelibrary.wiley.com/doi/pdfdirect/10.1111/bjet.13570","source":{"id":"https://openalex.org/S110346167","display_name":"British Journal of Educational Technology","issn_l":"0007-1013","issn":["0007-1013","1467-8535"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"British Journal of Educational Technology","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1111/bjet.13570","is_oa":true,"landing_page_url":"https://doi.org/10.1111/bjet.13570","pdf_url":"https://onlinelibrary.wiley.com/doi/pdfdirect/10.1111/bjet.13570","source":{"id":"https://openalex.org/S110346167","display_name":"British Journal of Educational Technology","issn_l":"0007-1013","issn":["0007-1013","1467-8535"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"British Journal of Educational Technology","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4408320960.pdf","grobid_xml":"https://content.openalex.org/works/W4408320960.grobid-xml"},"referenced_works_count":63,"referenced_works":["https://openalex.org/W92323007","https://openalex.org/W1919216659","https://openalex.org/W1922655562","https://openalex.org/W1984253625","https://openalex.org/W2044040445","https://openalex.org/W2086180740","https://openalex.org/W2110100688","https://openalex.org/W2129187472","https://openalex.org/W2243425977","https://openalex.org/W2329442499","https://openalex.org/W2461994724","https://openalex.org/W2483532694","https://openalex.org/W2594978815","https://openalex.org/W2794022343","https://openalex.org/W2887719596","https://openalex.org/W2892456951","https://openalex.org/W2902452216","https://openalex.org/W2954996726","https://openalex.org/W2962369866","https://openalex.org/W2963216553","https://openalex.org/W2971296908","https://openalex.org/W2976637132","https://openalex.org/W2980214248","https://openalex.org/W3035682985","https://openalex.org/W3080512609","https://openalex.org/W3100043936","https://openalex.org/W4221161695","https://openalex.org/W4223507692","https://openalex.org/W4241116859","https://openalex.org/W4286897500","https://openalex.org/W4286958721","https://openalex.org/W4288059447","https://openalex.org/W4292779060","https://openalex.org/W4307934157","https://openalex.org/W4309773749","https://openalex.org/W4313063630","https://openalex.org/W4322801648","https://openalex.org/W4327810158","https://openalex.org/W4366582553","https://openalex.org/W4376116649","https://openalex.org/W4384659608","https://openalex.org/W4384918448","https://openalex.org/W4385572411","https://openalex.org/W4385573161","https://openalex.org/W4385682136","https://openalex.org/W4386518345","https://openalex.org/W4387936944","https://openalex.org/W4388092438","https://openalex.org/W4388093242","https://openalex.org/W4389519448","https://openalex.org/W4390041933","https://openalex.org/W4391136507","https://openalex.org/W4392120986","https://openalex.org/W4392773006","https://openalex.org/W4396650747","https://openalex.org/W4396827149","https://openalex.org/W4398192132","https://openalex.org/W4398784073","https://openalex.org/W4399612092","https://openalex.org/W4401306886","https://openalex.org/W4401819713","https://openalex.org/W4402670893","https://openalex.org/W4406073383"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W2334176212","https://openalex.org/W2186687002","https://openalex.org/W2143700524","https://openalex.org/W2998354981"],"abstract_inverted_index":{"Effective":[0],"educational":[1,206,240],"measurement":[2,207],"relies":[3],"heavily":[4],"on":[5],"the":[6,28,38,94,111,126,138,173,237,265,270],"curation":[7],"of":[8,24,31,40,68,88,121,131,195,227,239,269,273,276,318,351],"well\u2010designed":[9],"item":[10,13,135,139,289],"pools.":[11],"However,":[12,91],"calibration":[14],"is":[15,82,187,202],"time":[16],"consuming":[17],"and":[18,54,278],"costly,":[19],"requiring":[20],"a":[21,225,232,274,328,344],"sufficient":[22],"number":[23],"respondents":[25,144,301,320],"to":[26,59,66,84,100,108,180,198,220,283,292,302,347],"estimate":[27],"psychometric":[29,63,267],"properties":[30,64],"items.":[32,242,354],"In":[33],"this":[34,98,191,259],"study,":[35],"we":[36,92],"explore":[37],"potential":[39],"six":[41],"different":[42],"large":[43],"language":[44,214],"models":[45,215],"(LLMs;":[46],"GPT\u20103.5,":[47],"GPT\u20104,":[48],"Llama":[49,51],"2,":[50],"3,":[52],"Gemini\u2010Pro":[53],"Cohere":[55],"Command":[56],"R":[57],"Plus)":[58],"generate":[60],"responses":[61,197],"with":[62,152,166],"comparable":[65],"those":[67,293],"human":[69,115,154,196,222,304],"respondents.":[70,298],"Results":[71],"indicate":[72],"that":[73,81,87,118,288,324],"some":[74],"LLMs":[75,95,122,280],"exhibit":[76],"proficiency":[77,103],"in":[78,97,114,205,224],"College":[79],"Algebra":[80],"similar":[83,291],"or":[85,338],"exceeds":[86],"college":[89,132],"students.":[90,133],"find":[93],"used":[96],"study":[99,286],"have":[101,145,217],"narrow":[102],"distributions,":[104],"limiting":[105],"their":[106,153,163],"ability":[107,128,271],"fully":[109],"mimic":[110],"variability":[112],"observed":[113],"respondents,":[116],"but":[117],"an":[119,210,249],"ensemble":[120],"can":[123],"better":[124],"approximate":[125],"broader":[127],"distribution":[129,272],"typical":[130],"Utilizing":[134],"response":[136,305],"theory,":[137],"parameters":[140,290],"calibrated":[141,155],"by":[142,295,321],"LLM":[143,300,319],"high":[146],"correlations":[147],"(eg,":[148],"&gt;0.8":[149],"for":[150,162,235,252,311,331,335],"GPT\u20103.5)":[151],"counterparts.":[156],"Several":[157],"augmentation":[158,244],"strategies":[159],"are":[160],"evaluated":[161],"relative":[164],"performance,":[165],"resampling":[167],"methods":[168],"proving":[169],"most":[170],"effective,":[171],"enhancing":[172,253],"Spearman":[174],"correlation":[175],"from":[176],"0.89":[177],"(human":[178],"only)":[179],"0.93":[181],"(augmented":[182],"human).":[183],"Practitioner":[184],"notes":[185],"What":[186,258],"already":[188],"known":[189],"about":[190],"topic":[192],"The":[193,285,315],"collection":[194],"candidate":[199],"test":[200],"items":[201,334],"common":[203],"practice":[204,312],"when":[208],"designing":[209],"assessment":[211,241],"tool.":[212],"Large":[213],"(LLMs)":[216],"been":[218,248],"found":[219],"rival":[221],"abilities":[223],"variety":[226,275],"subject":[228],"areas,":[229],"making":[230],"them":[231],"low\u2010cost":[233,329],"option":[234,330],"testing":[236],"efficacy":[238],"Data":[243],"using":[245],"AI":[246],"has":[247],"effective":[250],"strategy":[251],"machine":[254],"learning":[255],"model":[256],"performance.":[257],"paper":[260,263],"adds":[261],"This":[262,341],"provides":[264],"first":[266],"analysis":[268],"open\u2010source":[277],"proprietary":[279],"as":[281],"compared":[282],"humans.":[284],"finds":[287],"produced":[294],"50":[296],"undergraduate":[297],"Using":[299],"augment":[303],"data":[306],"yields":[307],"mixed":[308],"results.":[309],"Implications":[310],"and/or":[313],"policy":[314],"moderate":[316],"performance":[317],"themselves":[322],"suggests":[323],"they":[325],"could":[326],"provide":[327],"curating":[332],"quality":[333],"low\u2010stakes":[336],"formative":[337],"summative":[339],"assessments.":[340],"methodology":[342],"offers":[343],"scalable":[345],"way":[346],"evaluate":[348],"vast":[349],"amounts":[350],"generative":[352],"AI\u2010produced":[353]},"counts_by_year":[{"year":2026,"cited_by_count":10},{"year":2025,"cited_by_count":15}],"updated_date":"2026-06-20T22:02:38.213706","created_date":"2025-10-10T00:00:00"}