{"id":"https://openalex.org/W7203737890","doi":"https://doi.org/10.1145/3841465","title":"GenTREC : The First Test Collection Generated by Large Language Models for Evaluating Information Retrieval Systems","display_name":"GenTREC : The First Test Collection Generated by Large Language Models for Evaluating Information Retrieval Systems","publication_year":2026,"publication_date":"2026-08-19","ids":{"openalex":"https://openalex.org/W7203737890","doi":"https://doi.org/10.1145/3841465"},"language":"en","primary_location":{"id":"doi:10.1145/3841465","doi":"https://doi.org/10.1145/3841465","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3841465","pdf_url":null,"source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["abdc-c","anvur-class-a","anvur-scientific","ccf-a","dongbi-a","jufo-3","norway-2","poland-140","russia-white-list-1","vabb-shw"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1145/3841465","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5015705660","display_name":"Mehmet Deniz T\u00fcrkmen","orcid":"https://orcid.org/0009-0003-8705-7770","observed_orcids":["https://orcid.org/0009-0003-8705-7770"]},"institutions":[{"id":"https://openalex.org/I4210101898","display_name":"GESIS - Leibniz Institute for the Social Sciences","ror":"https://ror.org/018afyw53","country_code":"DE","type":"facility","lineage":["https://openalex.org/I315704651","https://openalex.org/I4210101898"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Mehmet Deniz T\u00fcrkmen","raw_affiliation_strings":["Knowledge Technologies for the Social Sciences, GESIS\u2014Leibniz Institute for the Social Sciences, Cologne, Germany"],"raw_orcid":"https://orcid.org/0009-0003-8705-7770","affiliations":[{"raw_affiliation_string":"Knowledge Technologies for the Social Sciences, GESIS\u2014Leibniz Institute for the Social Sciences, Cologne, Germany","institution_ids":["https://openalex.org/I4210101898"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086826049","display_name":"M\u00fccahid Kutlu","orcid":"https://orcid.org/0000-0002-5660-4992","observed_orcids":["https://orcid.org/0000-0002-5660-4992"]},"institutions":[{"id":"https://openalex.org/I60342839","display_name":"Qatar University","ror":"https://ror.org/00yhnba62","country_code":"QA","type":"education","lineage":["https://openalex.org/I60342839"]}],"countries":["QA"],"is_corresponding":false,"raw_author_name":"Mucahid Kutlu","raw_affiliation_strings":["Computer Science and Engineering Department, Qatar University, Doha, Qatar"],"raw_orcid":"https://orcid.org/0000-0002-5660-4992","affiliations":[{"raw_affiliation_string":"Computer Science and Engineering Department, Qatar University, Doha, Qatar","institution_ids":["https://openalex.org/I60342839"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014411432","display_name":"Bahad\u0131r ALTUN","orcid":"https://orcid.org/0000-0002-6503-7109","observed_orcids":["https://orcid.org/0000-0002-6503-7109","https://orcid.org/0009-0006-5513-8471"]},"institutions":[{"id":"https://openalex.org/I13236232","display_name":"TOBB University of Economics and Technology","ror":"https://ror.org/03ewx7v96","country_code":"TR","type":"education","lineage":["https://openalex.org/I13236232"]}],"countries":["TR"],"is_corresponding":false,"raw_author_name":"Bahadir Altun","raw_affiliation_strings":["Department of Computer Engineering, TOBB University of Economics and Technology, Ankara, Turkey"],"raw_orcid":"https://orcid.org/0009-0006-5513-8471","affiliations":[{"raw_affiliation_string":"Department of Computer Engineering, TOBB University of Economics and Technology, Ankara, Turkey","institution_ids":["https://openalex.org/I13236232"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5115799298","display_name":"Gokalp Cosgun","orcid":"https://orcid.org/0009-0005-7497-8380","observed_orcids":["https://orcid.org/0009-0005-7497-8380"]},"institutions":[{"id":"https://openalex.org/I13236232","display_name":"TOBB University of Economics and Technology","ror":"https://ror.org/03ewx7v96","country_code":"TR","type":"education","lineage":["https://openalex.org/I13236232"]}],"countries":["TR"],"is_corresponding":false,"raw_author_name":"Gokalp Cosgun","raw_affiliation_strings":["Department of Computer Engineering, TOBB University of Economics and Technology, Ankara, Turkey"],"raw_orcid":"https://orcid.org/0009-0005-7497-8380","affiliations":[{"raw_affiliation_string":"Department of Computer Engineering, TOBB University of Economics and Technology, Ankara, Turkey","institution_ids":["https://openalex.org/I13236232"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":7.5119,"has_fulltext":true,"cited_by_count":5,"citation_normalized_percentile":{"value":0.97536318,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":97},"biblio":{"volume":"44","issue":"7","first_page":"1","last_page":"36"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.00019999999494757503,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9599999785423279},{"id":"https://openalex.org/keywords/information-retrieval-evaluation","display_name":"information retrieval evaluation","score":0.9589999914169312},{"id":"https://openalex.org/keywords/relevance-assessment","display_name":"relevance assessment","score":0.5249999761581421},{"id":"https://openalex.org/keywords/assessment-reliability","display_name":"assessment reliability","score":0.5109999775886536},{"id":"https://openalex.org/keywords/document-quality","display_name":"document quality","score":0.5730000138282776}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.9027000069618225},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.8440999984741211},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.8317000269889832},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6168000102043152},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5740000009536743},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.558899998664856},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5357999801635742},{"id":"https://openalex.org/C161156560","wikidata":"https://www.wikidata.org/wiki/Q1638872","display_name":"Document retrieval","level":2,"score":0.5242000222206116},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.48820000886917114},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.4300000071525574},{"id":"https://openalex.org/C169903167","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Test set","level":2,"score":0.4115000069141388},{"id":"https://openalex.org/C2778842860","wikidata":"https://www.wikidata.org/wiki/Q986551","display_name":"Cross-language information retrieval","level":3,"score":0.4056999981403351},{"id":"https://openalex.org/C86037889","wikidata":"https://www.wikidata.org/wiki/Q4330127","display_name":"Learning to rank","level":3,"score":0.3937999904155731},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.39079999923706055},{"id":"https://openalex.org/C90288658","wikidata":"https://www.wikidata.org/wiki/Q3318149","display_name":"Human\u2013computer information retrieval","level":3,"score":0.37700000405311584},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3465000092983246},{"id":"https://openalex.org/C89686163","wikidata":"https://www.wikidata.org/wiki/Q1187982","display_name":"Vector space model","level":2,"score":0.34279999136924744},{"id":"https://openalex.org/C97854310","wikidata":"https://www.wikidata.org/wiki/Q19541","display_name":"Search engine","level":2,"score":0.3222000002861023},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3084000051021576},{"id":"https://openalex.org/C2778698081","wikidata":"https://www.wikidata.org/wiki/Q179797","display_name":"Thesaurus","level":2,"score":0.2964000105857849},{"id":"https://openalex.org/C551230270","wikidata":"https://www.wikidata.org/wiki/Q4368942","display_name":"Data retrieval","level":2,"score":0.289900004863739},{"id":"https://openalex.org/C99016210","wikidata":"https://www.wikidata.org/wiki/Q5488129","display_name":"Query expansion","level":2,"score":0.28450000286102295},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.28290000557899475},{"id":"https://openalex.org/C2779532271","wikidata":"https://www.wikidata.org/wiki/Q445558","display_name":"Relevance feedback","level":4,"score":0.2750999927520752},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.27390000224113464},{"id":"https://openalex.org/C180198813","wikidata":"https://www.wikidata.org/wiki/Q121182","display_name":"Information system","level":2,"score":0.2728999853134155},{"id":"https://openalex.org/C21025794","wikidata":"https://www.wikidata.org/wiki/Q5141219","display_name":"Cognitive models of information retrieval","level":4,"score":0.2685000002384186},{"id":"https://openalex.org/C78999398","wikidata":"https://www.wikidata.org/wiki/Q2899888","display_name":"Information needs","level":2,"score":0.2565000057220459}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3841465","doi":"https://doi.org/10.1145/3841465","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3841465","pdf_url":null,"source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["abdc-c","anvur-class-a","anvur-scientific","ccf-a","dongbi-a","jufo-3","norway-2","poland-140","russia-white-list-1","vabb-shw"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2501.02408","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2501.02408","pdf_url":"https://arxiv.org/pdf/2501.02408","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2501.02408","doi":"https://doi.org/10.48550/arxiv.2501.02408","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2501.02408","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1145/3841465","doi":"https://doi.org/10.1145/3841465","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3841465","pdf_url":null,"source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["abdc-c","anvur-class-a","anvur-scientific","ccf-a","dongbi-a","jufo-3","norway-2","poland-140","russia-white-list-1","vabb-shw"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7203737890.pdf","grobid_xml":"https://content.openalex.org/works/W7203737890.grobid-xml"},"referenced_works_count":28,"referenced_works":["https://openalex.org/W119481552","https://openalex.org/W1507711477","https://openalex.org/W1824124613","https://openalex.org/W1970548730","https://openalex.org/W1976076261","https://openalex.org/W1988091129","https://openalex.org/W2069870183","https://openalex.org/W2090805977","https://openalex.org/W2102229948","https://openalex.org/W2163652601","https://openalex.org/W2758898480","https://openalex.org/W2990592470","https://openalex.org/W3004392977","https://openalex.org/W3100107515","https://openalex.org/W4221142221","https://openalex.org/W4221142817","https://openalex.org/W4242023143","https://openalex.org/W4322718191","https://openalex.org/W4382449327","https://openalex.org/W4384107234","https://openalex.org/W4391505434","https://openalex.org/W4396816204","https://openalex.org/W4399354968","https://openalex.org/W4399597996","https://openalex.org/W4399795632","https://openalex.org/W4405193806","https://openalex.org/W4408184531","https://openalex.org/W4411549495"],"related_works":["https://openalex.org/W2077005289","https://openalex.org/W4400526284","https://openalex.org/W4411549495","https://openalex.org/W3094927943","https://openalex.org/W2581240705","https://openalex.org/W4412377927","https://openalex.org/W4400374549","https://openalex.org/W4407780254","https://openalex.org/W1968831592","https://openalex.org/W4405761894"],"abstract_inverted_index":{"Building":[0],"test":[1,45,156],"collections":[2,33,157],"for":[3,61,158,175],"Information":[4],"Retrieval":[5],"evaluation":[6,141,188],"has":[7],"traditionally":[8],"been":[9,27],"a":[10,35,53,83,108,171],"resource-intensive":[11],"and":[12,110,122,140,162,184],"time-consuming":[13],"task":[14],"due":[15],"to":[16,71,79,87,129],"reliance":[17],"on":[18],"manual":[19,62],"relevance":[20,63,137],"judgments.":[21,64],"While":[22],"various":[23],"cost-effective":[24],"strategies":[25],"have":[26],"explored,":[28],"the":[29,43,59,180],"development":[30],"of":[31,113,134,182],"such":[32],"remains":[34],"significant":[36],"challenge.":[37],"This":[38],"article":[39],"introduces":[40],"GenTREC":[41,117,131,151,169],",":[42],"first":[44],"collection":[46,118],"built":[47],"entirely":[48],"from":[49],"documents":[50,67,121],"generated":[51,98],"by":[52],"Large":[54],"Language":[55],"Model":[56],"(LLM),":[57],"eliminating":[58],"need":[60],"Assuming":[65],"LLM-generated":[66],"are":[68,105],"inherently":[69],"relevant":[70,85],"their":[72],"prompts,":[73],"we":[74,96],"used":[75],"TREC":[76,155],"search":[77],"topics":[78],"generate":[80],"documents,":[81,100],"considering":[82],"document":[84,135],"only":[86],"its":[88],"generating":[89],"prompt.":[90],"To":[91],"introduce":[92],"realistic":[93],"retrieval":[94],"challenges,":[95],"also":[97],"non-relevant":[99],"ensuring":[101],"that":[102,146,168],"IR":[103,147,176,187],"systems":[104],"tested":[106],"against":[107],"diverse":[109],"robust":[111],"set":[112],"materials.":[114],"The":[115],"resulting":[116],"comprises":[119],"96,196":[120],"300":[123],"topics.":[124],"We":[125],"conducted":[126],"extensive":[127],"experiments":[128],"evaluate":[130],"in":[132],"terms":[133],"quality,":[136],"judgment":[138],"accuracy,":[139],"reliability.":[142],"Our":[143,165],"results":[144],"show":[145],"system":[148],"rankings":[149],"using":[150],"align":[152],"with":[153],"traditional":[154],"P@100,":[159],"MAP,":[160],"RPrec,":[161],"nDCG":[163],"metrics.":[164],"findings":[166],"demonstrate":[167],"offers":[170],"promising,":[172],"low-cost":[173],"alternative":[174],"evaluation,":[177],"significantly":[178],"reducing":[179],"burden":[181],"building":[183],"maintaining":[185],"future":[186],"resources.":[189]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":3}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-08-20T00:00:00"}