{"id":"https://openalex.org/W7161687037","doi":"https://doi.org/10.48550/arxiv.2605.18562","title":"Estimating Item Difficulty with Large Language Models as Experts","display_name":"Estimating Item Difficulty with Large Language Models as Experts","publication_year":2026,"publication_date":"2026-05-18","ids":{"openalex":"https://openalex.org/W7161687037","doi":"https://doi.org/10.48550/arxiv.2605.18562"},"language":"en","primary_location":{"id":"pmh:oai:dspace.library.uu.nl:1874/485185","is_oa":true,"landing_page_url":"https://dspace.library.uu.nl/handle/1874/485185","pdf_url":null,"source":{"id":"https://openalex.org/S4306401649","display_name":"Utrecht University Repository (Utrecht University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I193662353","host_organization_name":"Utrecht University","host_organization_lineage":["https://openalex.org/I193662353"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"/dk/atira/pure/researchoutput/researchoutputtypes/workingpaper/preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://dspace.library.uu.nl/handle/1874/485185","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136497527","display_name":"Diana Kolesnikova","orcid":null},"institutions":[{"id":"https://openalex.org/I193700539","display_name":"Tilburg University","ror":"https://ror.org/04b8v1s79","country_code":"NL","type":"education","lineage":["https://openalex.org/I193700539"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Kolesnikova, Diana","raw_affiliation_strings":["Department of Methodology and Statistics, Tilburg University, Tilburg, Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Methodology and Statistics, Tilburg University, Tilburg, Netherlands","institution_ids":["https://openalex.org/I193700539"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042032750","display_name":"Kirill Fedyanin","orcid":"https://orcid.org/0000-0003-0363-9195"},"institutions":[{"id":"https://openalex.org/I4210106484","display_name":"Center for Health, Exercise and Sport Sciences","ror":"https://ror.org/0129x4h77","country_code":"RS","type":"other","lineage":["https://openalex.org/I4210106484"]}],"countries":["RS"],"is_corresponding":false,"raw_author_name":"Fedyanin, Kirill","raw_affiliation_strings":["Smart Business Technologies, Belgrade, Serbia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Smart Business Technologies, Belgrade, Serbia","institution_ids":["https://openalex.org/I4210106484"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136475007","display_name":"Abe D. Hofman","orcid":null},"institutions":[{"id":"https://openalex.org/I4210108148","display_name":"Lear (Spain)","ror":"https://ror.org/01j7j0055","country_code":"ES","type":"company","lineage":["https://openalex.org/I4210108148","https://openalex.org/I82700718"]},{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["ES","NL"],"is_corresponding":false,"raw_author_name":"Hofman, Abe D.","raw_affiliation_strings":["Department of Psychological Methods, University of Amsterdam, Amsterdam, Netherlands","Prowise Learn, Amsterdam, Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Psychological Methods, University of Amsterdam, Amsterdam, Netherlands","institution_ids":["https://openalex.org/I887064364"]},{"raw_affiliation_string":"Prowise Learn, Amsterdam, Netherlands","institution_ids":["https://openalex.org/I4210108148"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008347841","display_name":"Matthieu Brinkhuis","orcid":"https://orcid.org/0000-0003-1054-6683"},"institutions":[{"id":"https://openalex.org/I193662353","display_name":"Utrecht University","ror":"https://ror.org/04pp8hn57","country_code":"NL","type":"education","lineage":["https://openalex.org/I193662353"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Brinkhuis, Matthieu J. S.","raw_affiliation_strings":["Department of Information and Computing Sciences, Utrecht University, Utrecht, Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Information and Computing Sciences, Utrecht University, Utrecht, Netherlands","institution_ids":["https://openalex.org/I193662353"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5136491923","display_name":"Maria Bolsinova","orcid":null},"institutions":[{"id":"https://openalex.org/I193700539","display_name":"Tilburg University","ror":"https://ror.org/04b8v1s79","country_code":"NL","type":"education","lineage":["https://openalex.org/I193700539"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Bolsinova, Maria","raw_affiliation_strings":["Department of Methodology and Statistics, Tilburg University, Tilburg, Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Methodology and Statistics, Tilburg University, Tilburg, Netherlands","institution_ids":["https://openalex.org/I193700539"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.762499988079071,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.762499988079071,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.10440000146627426,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.010999999940395355,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/judgement","display_name":"Judgement","score":0.746399998664856},{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"Pairwise comparison","score":0.6991000175476074},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.5357000231742859},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.4523000121116638},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.4523000121116638},{"id":"https://openalex.org/keywords/range","display_name":"Range (aeronautics)","score":0.451200008392334},{"id":"https://openalex.org/keywords/empirical-evidence","display_name":"Empirical evidence","score":0.41029998660087585}],"concepts":[{"id":"https://openalex.org/C2776548248","wikidata":"https://www.wikidata.org/wiki/Q12621536","display_name":"Judgement","level":2,"score":0.746399998664856},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.6991000175476074},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6168000102043152},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5921000242233276},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5662999749183655},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.5357000231742859},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.4523000121116638},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.4523000121116638},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.451200008392334},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4171000123023987},{"id":"https://openalex.org/C166052673","wikidata":"https://www.wikidata.org/wiki/Q83021","display_name":"Empirical evidence","level":2,"score":0.41029998660087585},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.4000000059604645},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.34700000286102295},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.32589998841285706},{"id":"https://openalex.org/C96250715","wikidata":"https://www.wikidata.org/wiki/Q965330","display_name":"Estimation","level":2,"score":0.32109999656677246},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.31290000677108765},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.30720001459121704},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.2930000126361847},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.2924000024795532},{"id":"https://openalex.org/C106347477","wikidata":"https://www.wikidata.org/wiki/Q5384228","display_name":"Equating","level":3,"score":0.28459998965263367},{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.2815000116825104},{"id":"https://openalex.org/C44083865","wikidata":"https://www.wikidata.org/wiki/Q3853443","display_name":"Mean reciprocal rank","level":2,"score":0.27250000834465027},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.2551000118255615}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:dspace.library.uu.nl:1874/485185","is_oa":true,"landing_page_url":"https://dspace.library.uu.nl/handle/1874/485185","pdf_url":null,"source":{"id":"https://openalex.org/S4306401649","display_name":"Utrecht University Repository (Utrecht University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I193662353","host_organization_name":"Utrecht University","host_organization_lineage":["https://openalex.org/I193662353"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"/dk/atira/pure/researchoutput/researchoutputtypes/workingpaper/preprint"},{"id":"doi:10.48550/arxiv.2605.18562","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.18562","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:dspace.library.uu.nl:1874/485185","is_oa":true,"landing_page_url":"https://dspace.library.uu.nl/handle/1874/485185","pdf_url":null,"source":{"id":"https://openalex.org/S4306401649","display_name":"Utrecht University Repository (Utrecht University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I193662353","host_organization_name":"Utrecht University","host_organization_lineage":["https://openalex.org/I193662353"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"/dk/atira/pure/researchoutput/researchoutputtypes/workingpaper/preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.5881432890892029,"display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Accurate":[0],"estimates":[1,114,149,162],"of":[2,108,183,203,214],"item":[3,96,171,240],"difficulty":[4,70,83,113,148,219],"are":[5,21],"essential":[6],"for":[7,15,69,85,188,238],"valid":[8],"assessment":[9],"and":[10,25,31,62,141,212,242],"effective":[11,246],"adaptive":[12],"learning.":[13],"However,":[14,53,206],"newly":[16,86],"created":[17,87],"tasks,":[18,176],"response":[19,92],"data":[20],"typically":[22],"unavailable.":[23],"Pretesting":[24],"expert":[26],"judgement":[27,129,199,224],"can":[28],"be":[29],"costly":[30],"slow,":[32],"while":[33],"machine":[34],"learning":[35,101],"methods":[36],"often":[37],"require":[38],"large":[39,47],"labelled":[40],"training":[41],"datasets.":[42],"Recent":[43],"work":[44],"suggests":[45],"that":[46],"language":[48],"models":[49],"(LLMs)":[50],"may":[51],"help.":[52],"there":[54],"is":[55],"limited":[56],"evidence":[57],"on":[58],"the":[59,103,180,184,201,222],"elicitation":[60],"procedures":[61],"prompt":[63],"configurations":[64,178],"used":[65,121],"to":[66,91,165],"emulate":[67],"experts":[68,190],"estimation.":[71],"This":[72],"study":[73,104,120,231],"addresses":[74],"this":[75],"gap":[76],"by":[77],"evaluating":[78],"three":[79,127],"off-the-shelf":[80],"LLMs":[81,233],"as":[82,116,234],"raters":[84],"items":[88,215],"without":[89],"access":[90],"data.":[93],"Using":[94],"an":[95,99],"bank":[97],"from":[98],"online":[100],"system,":[102],"examined":[105],"6":[106],"domains":[107],"primary-school":[109],"mathematics,":[110],"with":[111,152,169,216],"empirical":[112,117,153,170,218],"treated":[115],"reference.":[118],"The":[119,230],"a":[122,235],"full":[123],"factorial":[124],"design":[125],"crossing":[126],"factors:":[128],"format":[130],"(absolute":[131],"vs":[132,138,145],"pairwise),":[133],"decision":[134],"type":[135],"(hard":[136],"decisions":[137],"token-probability-based":[139],"estimates),":[140],"prompting":[142],"strategy":[143],"(zero-shot":[144],"few-shot).":[146],"LLM-derived":[147],"were":[150,210,220],"compared":[151],"difficulties":[154],"using":[155],"Spearman":[156],"rank":[157],"correlations.":[158],"Across":[159],"domains,":[160],"LLM-based":[161],"exhibited":[163],"moderate":[164],"strong":[166],"positive":[167],"correlations":[168],"difficulties.":[172],"For":[173],"simpler":[174],"arithmetic":[175],"some":[177],"approached":[179],"upper":[181],"end":[182],"accuracy":[185],"range":[186],"reported":[187],"human":[189],"in":[191,200],"previous":[192],"research.":[193],"Pairwise":[194],"comparison":[195],"consistently":[196],"outperformed":[197],"absolute":[198,223],"absence":[202],"additional":[204],"refinements.":[205],"when":[207],"token-level":[208],"probabilities":[209],"incorporated":[211],"examples":[213],"known":[217],"provided,":[221],"configuration":[225],"likewise":[226],"demonstrated":[227],"moderate-to-high":[228],"alignment.":[229],"positions":[232],"promising":[236],"tool":[237],"initial":[239],"calibration":[241],"offers":[243],"insights":[244],"into":[245],"workflow":[247],"configuration.":[248]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-05-20T00:00:00"}