{"id":"https://openalex.org/W4416017378","doi":"https://doi.org/10.1145/3746252.3760908","title":"Towards Understanding Bias in Synthetic Data for Evaluation","display_name":"Towards Understanding Bias in Synthetic Data for Evaluation","publication_year":2025,"publication_date":"2025-11-08","ids":{"openalex":"https://openalex.org/W4416017378","doi":"https://doi.org/10.1145/3746252.3760908"},"language":null,"primary_location":{"id":"doi:10.1145/3746252.3760908","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3746252.3760908","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 34th ACM International Conference on Information and Knowledge Management","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3746252.3760908","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5044972686","display_name":"Hossein A. Rahmani","orcid":"https://orcid.org/0000-0002-2779-4942"},"institutions":[{"id":"https://openalex.org/I4210128584","display_name":"The Alan Turing Institute","ror":"https://ror.org/035dkdb55","country_code":"GB","type":"facility","lineage":["https://openalex.org/I4210128584"]},{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Hossein A. Rahmani","raw_affiliation_strings":["University College London, London, United Kingdom and The Alan Turing Institute, London, United Kingdom"],"raw_orcid":"https://orcid.org/0000-0002-2779-4942","affiliations":[{"raw_affiliation_string":"University College London, London, United Kingdom and The Alan Turing Institute, London, United Kingdom","institution_ids":["https://openalex.org/I4210128584","https://openalex.org/I45129253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016903660","display_name":"Varsha Ramineni","orcid":"https://orcid.org/0000-0002-0330-3184"},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Varsha Ramineni","raw_affiliation_strings":["University College London, London, United Kingdom"],"raw_orcid":"https://orcid.org/0000-0002-0330-3184","affiliations":[{"raw_affiliation_string":"University College London, London, United Kingdom","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101716010","display_name":"Emine Yilmaz","orcid":"https://orcid.org/0000-0003-4734-4532"},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Emine Yilmaz","raw_affiliation_strings":["University College London, London, United Kingdom"],"raw_orcid":"https://orcid.org/0000-0003-4734-4532","affiliations":[{"raw_affiliation_string":"University College London, London, United Kingdom","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055132321","display_name":"Nick Craswell","orcid":"https://orcid.org/0000-0002-9351-8137"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I58610484","display_name":"Seattle University","ror":"https://ror.org/02jqc0m91","country_code":"US","type":"education","lineage":["https://openalex.org/I58610484"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nick Craswell","raw_affiliation_strings":["Microsoft, Seattle, USA"],"raw_orcid":"https://orcid.org/0000-0002-9351-8137","affiliations":[{"raw_affiliation_string":"Microsoft, Seattle, USA","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I58610484"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5048533217","display_name":"Bhaskar Mitra","orcid":"https://orcid.org/0000-0002-5270-5550"},"institutions":[{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Bhaskar Mitra","raw_affiliation_strings":["Research, Microsoft, Montreal, Canada"],"raw_orcid":"https://orcid.org/0000-0002-5270-5550","affiliations":[{"raw_affiliation_string":"Research, Microsoft, Montreal, Canada","institution_ids":["https://openalex.org/I4210164937"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":4.7179,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.95618251,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"5166","last_page":"5170"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9199000000953674,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9199000000953674,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.009999999776482582,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.007300000172108412,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.791100025177002},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.6880999803543091},{"id":"https://openalex.org/keywords/synthetic-data","display_name":"Synthetic data","score":0.6315000057220459},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.5841000080108643},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5579000115394592},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.44859999418258667}],"concepts":[{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.791100025177002},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7059999704360962},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.6880999803543091},{"id":"https://openalex.org/C160920958","wikidata":"https://www.wikidata.org/wiki/Q7662746","display_name":"Synthetic data","level":2,"score":0.6315000057220459},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.5841000080108643},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5579000115394592},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5072000026702881},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.46149998903274536},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.44859999418258667},{"id":"https://openalex.org/C169903167","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Test set","level":2,"score":0.42250001430511475},{"id":"https://openalex.org/C58489278","wikidata":"https://www.wikidata.org/wiki/Q1172284","display_name":"Data set","level":2,"score":0.38519999384880066},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3587999939918518},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3449999988079071},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.33399999141693115},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3068000078201294},{"id":"https://openalex.org/C16910744","wikidata":"https://www.wikidata.org/wiki/Q7705759","display_name":"Test data","level":2,"score":0.27959999442100525}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3746252.3760908","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3746252.3760908","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 34th ACM International Conference on Information and Knowledge Management","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3746252.3760908","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3746252.3760908","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 34th ACM International Conference on Information and Knowledge Management","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4583412171","display_name":"UKRI AI Centre for Doctoral Training in Foundational Artificial Intelligence","funder_award_id":"EP/S021566/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"}],"funders":[{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":9,"referenced_works":["https://openalex.org/W2009954908","https://openalex.org/W4284669679","https://openalex.org/W4385688511","https://openalex.org/W4389520342","https://openalex.org/W4400526284","https://openalex.org/W4400526908","https://openalex.org/W4401330297","https://openalex.org/W4407953544","https://openalex.org/W4410638169"],"related_works":[],"abstract_inverted_index":{"Test":[0],"collections":[1,19,78,89,119,147,204],"are":[2,125,148,229],"crucial":[3],"for":[4,17,96,150,208],"evaluating":[5],"Information":[6],"Retrieval":[7],"(IR)":[8],"systems.":[9],"Creating":[10],"a":[11,34,183],"diverse":[12],"set":[13],"of":[14,116,158,179,194],"user":[15],"queries":[16,65],"these":[18],"can":[20],"be":[21,94,206,218],"challenging,":[22],"and":[23,39,164,227],"obtaining":[24],"relevance":[25],"judgments,":[26],"which":[27],"indicate":[28],"how":[29],"well":[30],"retrieved":[31],"documents":[32,67],"match":[33],"query,":[35],"is":[36,79,102],"often":[37],"costly":[38],"resource-intensive.":[40],"Recently,":[41],"generating":[42],"synthetic":[43,64,76,87,117,129,202],"datasets":[44],"using":[45,72,121,182,201],"Large":[46],"Language":[47],"Models":[48],"(LLMs)":[49],"has":[50,59],"gained":[51],"attention":[52],"in":[53,161,197,221],"various":[54],"applications.":[55],"While":[56],"previous":[57],"work":[58,84],"used":[60,95,126,149],"LLMs":[61,73,124],"to":[62,68,74,93,104,127],"generate":[63,128],"or":[66,132],"improve":[69],"ranking":[70],"models,":[71],"create":[75],"test":[77,88,118,146,203],"still":[80],"relatively":[81],"unexplored.":[82],"Previous":[83],"showed":[85],"that":[86,141,190],"have":[90,170],"the":[91,114,138,156,166,177,192],"potential":[92,139],"system":[97,172,212,224],"evaluation,":[98],"however,":[99],"more":[100],"analysis":[101,188],"needed":[103],"validate":[105,176],"this":[106,109],"claim.":[107],"In":[108,134],"paper,":[110],"we":[111,136],"thoroughly":[112],"investigate":[113],"reliability":[115],"constructed":[120],"LLMs,":[122],"where":[123],"queries,":[130],"labels,":[131],"both.":[133],"particular,":[135],"examine":[137],"biases":[140],"might":[142,169],"occur":[143],"when":[144],"such":[145,159,180],"evaluation.":[151,173],"We":[152,174],"first":[153],"empirically":[154],"show":[155],"presence":[157,178],"bias":[160,181,195],"evaluation":[162,198],"results":[163,199],"analyse":[165],"effects":[167],"it":[168],"on":[171],"further":[175],"linear":[184],"mixed-effects":[185],"model.":[186],"Our":[187],"shows":[189],"while":[191],"effect":[193,215],"present":[196],"obtained":[200],"could":[205],"significant,":[207],"e.g.":[209],"computing":[210],"absolute":[211],"performance,":[213],"its":[214],"may":[216],"not":[217],"as":[219],"significant":[220],"comparing":[222],"relative":[223],"performance.":[225],"Codes":[226],"data":[228],"available":[230],"at:":[231],"https://github.com/rahmanidashti/BiasSyntheticData":[232]},"counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-11-08T00:00:00"}