{"id":"https://openalex.org/W7160559280","doi":"https://doi.org/10.48550/arxiv.2605.04298","title":"Towards Self-Referential Analytic Assessment: A Profile-Based Approach to L2 Writing Evaluation with LLMs","display_name":"Towards Self-Referential Analytic Assessment: A Profile-Based Approach to L2 Writing Evaluation with LLMs","publication_year":2026,"publication_date":"2026-05-05","ids":{"openalex":"https://openalex.org/W7160559280","doi":"https://doi.org/10.48550/arxiv.2605.04298"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.04298","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.04298","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.04298","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5015260062","display_name":"Stefano Bann\u00f2","orcid":"https://orcid.org/0000-0002-2799-0601"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bann\u00f2, Stefano","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133763768","display_name":"Kate Knill","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Knill, Kate","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5121270832","display_name":"Mark Gales","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gales, Mark","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.45100000500679016,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.45100000500679016,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11587","display_name":"Second Language Acquisition and Learning","score":0.08709999918937683,"subfield":{"id":"https://openalex.org/subfields/3204","display_name":"Developmental and Educational Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.04820000007748604,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/strengths-and-weaknesses","display_name":"Strengths and weaknesses","score":0.7656000256538391},{"id":"https://openalex.org/keywords/rasch-model","display_name":"Rasch model","score":0.6535999774932861},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.5521000027656555},{"id":"https://openalex.org/keywords/component","display_name":"Component (thermodynamics)","score":0.4309000074863434},{"id":"https://openalex.org/keywords/readability","display_name":"Readability","score":0.4075999855995178},{"id":"https://openalex.org/keywords/value","display_name":"Value (mathematics)","score":0.35190001130104065},{"id":"https://openalex.org/keywords/writing-assessment","display_name":"Writing assessment","score":0.3167000114917755},{"id":"https://openalex.org/keywords/language-proficiency","display_name":"Language proficiency","score":0.2955999970436096}],"concepts":[{"id":"https://openalex.org/C63882131","wikidata":"https://www.wikidata.org/wiki/Q17122954","display_name":"Strengths and weaknesses","level":2,"score":0.7656000256538391},{"id":"https://openalex.org/C101266164","wikidata":"https://www.wikidata.org/wiki/Q2131821","display_name":"Rasch model","level":2,"score":0.6535999774932861},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.5521000027656555},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.5407000184059143},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.4309000074863434},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4237000048160553},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.4174000024795532},{"id":"https://openalex.org/C2778143727","wikidata":"https://www.wikidata.org/wiki/Q1820650","display_name":"Readability","level":2,"score":0.4075999855995178},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4002000093460083},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.35190001130104065},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.33309999108314514},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.32330000400543213},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.31839999556541443},{"id":"https://openalex.org/C2777898490","wikidata":"https://www.wikidata.org/wiki/Q17157236","display_name":"Writing assessment","level":2,"score":0.3167000114917755},{"id":"https://openalex.org/C75630572","wikidata":"https://www.wikidata.org/wiki/Q538904","display_name":"Applied psychology","level":1,"score":0.3142000138759613},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.31130000948905945},{"id":"https://openalex.org/C2778883600","wikidata":"https://www.wikidata.org/wiki/Q2390977","display_name":"Language proficiency","level":2,"score":0.2955999970436096},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.2903999984264374},{"id":"https://openalex.org/C3018395757","wikidata":"https://www.wikidata.org/wiki/Q1379672","display_name":"Evaluation methods","level":2,"score":0.2872999906539917},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.28619998693466187},{"id":"https://openalex.org/C171606756","wikidata":"https://www.wikidata.org/wiki/Q506132","display_name":"Psychometrics","level":2,"score":0.2745000123977661},{"id":"https://openalex.org/C2776818064","wikidata":"https://www.wikidata.org/wiki/Q829903","display_name":"Agreement","level":2,"score":0.2678000032901764},{"id":"https://openalex.org/C88548561","wikidata":"https://www.wikidata.org/wiki/Q347599","display_name":"sort","level":2,"score":0.26330000162124634},{"id":"https://openalex.org/C19875794","wikidata":"https://www.wikidata.org/wiki/Q1207340","display_name":"Item response theory","level":3,"score":0.258899986743927},{"id":"https://openalex.org/C199075045","wikidata":"https://www.wikidata.org/wiki/Q762856","display_name":"Frequency","level":2,"score":0.25220000743865967},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.2502000033855438}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.04298","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.04298","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.04298","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.04298","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.8943781852722168,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Automated":[0],"essay":[1],"scoring":[2,136],"(AES)":[3],"research":[4],"often":[5],"relies":[6],"on":[7,70,84],"rank-based":[8,191],"correlation":[9],"metrics":[10,17],"to":[11,102,116,158],"validate":[12],"analytic":[13,23,127,135,194],"assessment.":[14],"However,":[15],"such":[16],"obscure":[18],"both":[19],"intrinsic":[20],"intercorrelations":[21],"among":[22],"dimensions":[24],"that":[25,68,155],"arise":[26],"from":[27],"the":[28,85,134,188,198],"structure":[29],"of":[30,138,190,200],"writing":[31,94],"proficiency":[32,171],"itself":[33],"and":[34,74,98,120,129,142,196,206],"halo":[35],"effects,":[36],"whereby":[37],"holistic":[38,130],"impressions":[39],"bleed":[40],"into":[41],"fine-grained":[42],"component":[43],"scores.":[44],"As":[45],"a":[46,52,62,90,149],"result,":[47],"high":[48],"correlations":[49],"may":[50],"mask":[51],"system's":[53],"true":[54],"diagnostic":[55],"behaviour.":[56],"In":[57],"this":[58],"study,":[59],"we":[60,111],"propose":[61],"novel":[63],"self-referential":[64],"assessment":[65,195],"evaluation":[66,192],"framework":[67],"focuses":[69],"identifying":[71,164,179],"intra-learner":[72],"strengths":[73,181],"weaknesses":[75,166],"rather":[76],"than":[77],"assessing":[78,205],"inter-learner":[79],"rankings.":[80],"We":[81,132],"conduct":[82],"experiments":[83],"publicly":[86],"available":[87],"ICNALE":[88],"GRA,":[89],"uniquely":[91],"dense":[92],"second-language":[93],"dataset":[95],"annotated":[96],"holistically":[97],"analytically":[99],"by":[100],"up":[101],"80":[103],"trained":[104],"raters.":[105],"To":[106],"obtain":[107],"reliable":[108],"reference":[109],"scores,":[110],"apply":[112],"two-facet":[113],"Rasch":[114],"modelling":[115],"calibrate":[117],"rater":[118],"severity":[119],"derive":[121],"fair":[122],"average":[123],"scores":[124],"across":[125,169],"ten":[126],"aspects":[128],"proficiency.":[131],"compare":[133],"performance":[137],"human":[139,161,174],"operational":[140],"raters":[141,162,175],"three":[143],"large":[144],"language":[145],"models":[146],"(LLMs)":[147],"in":[148,163,209],"zero-shot":[150],"setting.":[151],"Our":[152],"results":[153],"show":[154],"LLMs":[156,208],"tend":[157],"outperform":[159],"single":[160],"relative":[165,180],"(negative":[167],"feedback)":[168],"several":[170],"aspects,":[172],"while":[173],"remain":[176],"stronger":[177],"at":[178],"(positive":[182],"feedback).":[183],"Overall,":[184],"our":[185],"findings":[186],"highlight":[187],"limitations":[189],"for":[193,204],"demonstrate":[197],"value":[199],"intra-learner,":[201],"profile-based":[202],"methods":[203],"deploying":[207],"AES.":[210]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-08T00:00:00"}