{"id":"https://openalex.org/W4412975859","doi":"https://doi.org/10.1145/3698205.3729553","title":"Assessing Reliability in AI-Powered Learning Systems with A/A Tests","display_name":"Assessing Reliability in AI-Powered Learning Systems with A/A Tests","publication_year":2025,"publication_date":"2025-07-17","ids":{"openalex":"https://openalex.org/W4412975859","doi":"https://doi.org/10.1145/3698205.3729553"},"language":"en","primary_location":{"id":"doi:10.1145/3698205.3729553","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3698205.3729553","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twelfth ACM Conference on Learning @ Scale","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://pure.uva.nl/ws/files/267186503/3698205.3729553.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Lilian Ye","orcid":"https://orcid.org/0009-0008-6394-8065"},"institutions":[{"id":"https://openalex.org/I4210108148","display_name":"Lear (Spain)","ror":"https://ror.org/01j7j0055","country_code":"ES","type":"company","lineage":["https://openalex.org/I4210108148","https://openalex.org/I82700718"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Lilian Ye","raw_affiliation_strings":["Prowise Learn, Amsterdam, Netherlands and Psychological Methods, University of Amsterdam, Amsterdam, Netherlands"],"raw_orcid":"https://orcid.org/0009-0008-6394-8065","affiliations":[{"raw_affiliation_string":"Prowise Learn, Amsterdam, Netherlands and Psychological Methods, University of Amsterdam, Amsterdam, Netherlands","institution_ids":["https://openalex.org/I4210108148"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056712523","display_name":"Alexander Olof Savi","orcid":"https://orcid.org/0000-0002-9271-7476"},"institutions":[{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Alexander O. Savi","raw_affiliation_strings":["Psychological Methods, University of Amsterdam, Amsterdam, Netherlands"],"raw_orcid":"https://orcid.org/0000-0002-9271-7476","affiliations":[{"raw_affiliation_string":"Psychological Methods, University of Amsterdam, Amsterdam, Netherlands","institution_ids":["https://openalex.org/I887064364"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5053996683","display_name":"Abe Dirk Hofman","orcid":"https://orcid.org/0000-0003-4269-5296"},"institutions":[{"id":"https://openalex.org/I4210108148","display_name":"Lear (Spain)","ror":"https://ror.org/01j7j0055","country_code":"ES","type":"company","lineage":["https://openalex.org/I4210108148","https://openalex.org/I82700718"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Abe D. Hofman","raw_affiliation_strings":["Psychological Methods, University of Amsterdam, Amsterdam, Netherlands and Prowise Learn, Amsterdam, Netherlands"],"raw_orcid":"https://orcid.org/0000-0003-4269-5296","affiliations":[{"raw_affiliation_string":"Psychological Methods, University of Amsterdam, Amsterdam, Netherlands and Prowise Learn, Amsterdam, Netherlands","institution_ids":["https://openalex.org/I4210108148"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.2524,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.77656546,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"13","last_page":"23"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9695000052452087,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9695000052452087,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10809","display_name":"Occupational Health and Safety Research","score":0.9595999717712402,"subfield":{"id":"https://openalex.org/subfields/3614","display_name":"Radiological and Ultrasound Technology"},"field":{"id":"https://openalex.org/fields/36","display_name":"Health Professions"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9383999705314636,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.6805554628372192},{"id":"https://openalex.org/keywords/reliability-engineering","display_name":"Reliability engineering","score":0.6743351817131042},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6491968631744385},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.34413450956344604},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3413431644439697},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.2192637324333191}],"concepts":[{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.6805554628372192},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.6743351817131042},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6491968631744385},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.34413450956344604},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3413431644439697},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.2192637324333191},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3698205.3729553","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3698205.3729553","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twelfth ACM Conference on Learning @ Scale","raw_type":"proceedings-article"},{"id":"pmh:oai:dare.uva.nl:openaire_cris_publications/d55629ff-ae17-4e5b-8078-7a3c1892ac87","is_oa":true,"landing_page_url":"https://handle.uba.uva.nl/personal/pure/en/publications/assessing-reliability-in-aipowered-learning-systems-with-aa-tests(d55629ff-ae17-4e5b-8078-7a3c1892ac87).html","pdf_url":"https://pure.uva.nl/ws/files/267186503/3698205.3729553.pdf","source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Ye, L, Savi, A O & Hofman, A D 2025, Assessing Reliability in AI-Powered Learning Systems with A/A Tests. in L@S '25 : Proceedings of the Twelfth ACM Conference on Learning @ Scale : July 21-23, 2025, Palermo, Italy. Association for Computing Machinery, New York, NY, pp. 13-23, 12th ACM Conference on Learning @ Scale, L@S 2025, Palermo, Italy, 21/07/25. https://doi.org/10.1145/3698205.3729553","raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"pmh:oai:dare.uva.nl:publications/d55629ff-ae17-4e5b-8078-7a3c1892ac87","is_oa":true,"landing_page_url":"https://hdl.handle.net/11245.1/d55629ff-ae17-4e5b-8078-7a3c1892ac87","pdf_url":null,"source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Ye, L, Savi, A O & Hofman, A D 2025, Assessing Reliability in AI-Powered Learning Systems with A/A Tests. in L@S '25 : Proceedings of the Twelfth ACM Conference on Learning @ Scale : July 21-23, 2025, Palermo, Italy. Association for Computing Machinery, New York, NY, pp. 13-23, 12th ACM Conference on Learning @ Scale, L@S 2025, Palermo, Italy, 21/07/25. https://doi.org/10.1145/3698205.3729553","raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":{"id":"pmh:oai:dare.uva.nl:openaire_cris_publications/d55629ff-ae17-4e5b-8078-7a3c1892ac87","is_oa":true,"landing_page_url":"https://handle.uba.uva.nl/personal/pure/en/publications/assessing-reliability-in-aipowered-learning-systems-with-aa-tests(d55629ff-ae17-4e5b-8078-7a3c1892ac87).html","pdf_url":"https://pure.uva.nl/ws/files/267186503/3698205.3729553.pdf","source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Ye, L, Savi, A O & Hofman, A D 2025, Assessing Reliability in AI-Powered Learning Systems with A/A Tests. in L@S '25 : Proceedings of the Twelfth ACM Conference on Learning @ Scale : July 21-23, 2025, Palermo, Italy. Association for Computing Machinery, New York, NY, pp. 13-23, 12th ACM Conference on Learning @ Scale, L@S 2025, Palermo, Italy, 21/07/25. https://doi.org/10.1145/3698205.3729553","raw_type":"info:eu-repo/semantics/conferenceObject"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3905554197","display_name":null,"funder_award_id":"VI.Veni.211G.016","funder_id":"https://openalex.org/F4320323817","funder_display_name":"Universitas Brawijaya"}],"funders":[{"id":"https://openalex.org/F4320321800","display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek","ror":"https://ror.org/04jsz6e67"},{"id":"https://openalex.org/F4320323817","display_name":"Universitas Brawijaya","ror":"https://ror.org/01wk3d929"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412975859.pdf","grobid_xml":"https://content.openalex.org/works/W4412975859.grobid-xml"},"referenced_works_count":24,"referenced_works":["https://openalex.org/W973843712","https://openalex.org/W1597864774","https://openalex.org/W1981244178","https://openalex.org/W2069003154","https://openalex.org/W2084398551","https://openalex.org/W2104942079","https://openalex.org/W2160092878","https://openalex.org/W2160116727","https://openalex.org/W2162602057","https://openalex.org/W2199837949","https://openalex.org/W2335612402","https://openalex.org/W2341021244","https://openalex.org/W2781352358","https://openalex.org/W2885047664","https://openalex.org/W3045429925","https://openalex.org/W3150893739","https://openalex.org/W3214027242","https://openalex.org/W4300879629","https://openalex.org/W4301344865","https://openalex.org/W4312062489","https://openalex.org/W4389365236","https://openalex.org/W4390634198","https://openalex.org/W4395005032","https://openalex.org/W4411392672"],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W4387369504","https://openalex.org/W2033512842","https://openalex.org/W4394896187","https://openalex.org/W3170094116","https://openalex.org/W4233600955","https://openalex.org/W4386462264","https://openalex.org/W4322734194","https://openalex.org/W3116237489"],"abstract_inverted_index":{"The":[0],"rapid":[1],"evolution":[2],"of":[3,28,39,80,99,115,126,153,162,167,184,190,240],"Artificial":[4],"Intelligence":[5],"(AI)":[6],"has":[7],"expanded":[8],"access":[9],"to":[10,20,47,59,165,201],"large-scale":[11,103,132,241],"online":[12,133],"adaptive":[13],"learning":[14,23,50,71,83,117,135,243,263],"systems.":[15,244],"Such":[16],"AI-powered":[17,116,242],"systems":[18,41,87],"strive":[19],"deliver":[21],"personalized":[22],"experiences,":[24],"often":[25],"by":[26,122,140],"means":[27],"advanced":[29],"algorithms":[30],"that":[31,88,137,218],"continuously":[32,90],"model":[33,128,221],"learner":[34],"behavior.":[35],"Ensuring":[36],"the":[37,78,97,113,124,150,163,176,182,188,195,219,226,238,255],"reliability":[38,79,114,125,183,239],"these":[40,81],"is":[42,55,138],"fundamental,":[43],"otherwise":[44],"their":[45,199],"ability":[46,192,202],"optimize":[48],"individual":[49],"paths":[51],"and":[52,92,143,158,198,249],"inform":[53],"decision-making":[54],"undermined.":[56],"In":[57],"order":[58],"trust":[60],"such":[61],"systems,":[62],"learners":[63],"with":[64,106],"identical":[65,107,169],"profiles":[66],"should":[67],"follow":[68],"highly":[69],"similar":[70],"trajectories.":[72],"But":[73],"how":[74],"can":[75],"we":[76],"evaluate":[77],"dynamic":[82,262],"environments,":[84],"especially":[85],"in":[86,110,130,175,194,212,261],"are":[89,223],"developed":[91,156],"updated?":[93],"This":[94],"paper":[95],"demonstrates":[96],"effectiveness":[98],"A/A":[100,230],"testing":[101,231],"---":[102,109],"double-blind":[104],"experiments":[105],"conditions":[108],"systematically":[111],"evaluating":[112],"environments.":[118,264],"We":[119,148,179,245],"illustrate":[120],"this":[121],"assessing":[123,237],"student":[127,191,213,220],"parameters":[129,222],"a":[131,141,154,233],"arithmetic":[134,206],"platform":[136],"driven":[139],"well-studied":[142],"powerful":[144],"explainable":[145],"AI":[146],"algorithm.":[147],"duplicated":[149],"item":[151,185],"bank":[152],"newly":[155],"game":[157],"randomly":[159],"assigned":[160],"50%":[161],"players":[164],"one":[166],"two":[168,227],"versions,":[170],"which":[171],"were":[172],"launched":[173],"simultaneously":[174],"live":[177],"environment.":[178],"then":[180],"analyzed":[181],"difficulty":[186],"convergence,":[187],"stability":[189],"estimates":[193,203],"new":[196],"game,":[197],"relationship":[200],"from":[204],"other":[205],"games,":[207],"as":[208,210,232],"well":[209],"patterns":[211],"errors.":[214],"Our":[215],"results":[216],"indicate":[217],"stable":[224],"across":[225],"variants,":[228],"highlighting":[229],"valuable":[234],"tool":[235],"for":[236,253],"discuss":[246],"its":[247,259],"advantages":[248],"suggest":[250],"future":[251],"directions":[252],"adapting":[254],"approach,":[256],"while":[257],"considering":[258],"relevance":[260]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-08-01T09:00:35.917206","created_date":"2025-10-10T00:00:00"}