{"id":"https://openalex.org/W7161491309","doi":"https://doi.org/10.5281/zenodo.20256463","title":"SOCRATES-300K","display_name":"SOCRATES-300K","publication_year":2026,"publication_date":"2026-05-17","ids":{"openalex":"https://openalex.org/W7161491309","doi":"https://doi.org/10.5281/zenodo.20256463"},"language":null,"primary_location":{"id":"doi:10.5281/zenodo.20256463","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.20256463","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Dataset"},"type":"dataset","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.5281/zenodo.20256463","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5116269534","display_name":"Emanuele Ricco","orcid":null},"institutions":[{"id":"https://openalex.org/I71920554","display_name":"King Abdullah University of Science and Technology","ror":"https://ror.org/01q3tbs38","country_code":"SA","type":"education","lineage":["https://openalex.org/I71920554"]}],"countries":["SA"],"is_corresponding":false,"raw_author_name":"Emanuele Ricco","raw_affiliation_strings":["King Abdullah University of Science and Technology"],"raw_orcid":"https://orcid.org/0009-0007-5668-6688","affiliations":[{"raw_affiliation_string":"King Abdullah University of Science and Technology","institution_ids":["https://openalex.org/I71920554"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035147508","display_name":"E. Onofri","orcid":"https://orcid.org/0000-0001-8391-2563"},"institutions":[{"id":"https://openalex.org/I119003972","display_name":"Roma Tre University","ror":"https://ror.org/05vf0dg29","country_code":"IT","type":"education","lineage":["https://openalex.org/I119003972"]},{"id":"https://openalex.org/I4210106125","display_name":"Istituto per le Applicazioni del Calcolo Mauro Picone","ror":"https://ror.org/00ygy3d85","country_code":"IT","type":"facility","lineage":["https://openalex.org/I4210106125","https://openalex.org/I4210155236"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Onofri, Elia","raw_affiliation_strings":["Istituto per le Applicazioni del Calcolo Mauro Picone","Roma Tre University"],"raw_orcid":"https://orcid.org/0000-0001-8391-2563","affiliations":[{"raw_affiliation_string":"Istituto per le Applicazioni del Calcolo Mauro Picone","institution_ids":["https://openalex.org/I4210106125"]},{"raw_affiliation_string":"Roma Tre University","institution_ids":["https://openalex.org/I119003972"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136349104","display_name":"Lorenzo Cima","orcid":"https://orcid.org/0009-0001-5815-0437"},"institutions":[{"id":"https://openalex.org/I108290504","display_name":"University of Pisa","ror":"https://ror.org/03ad39j10","country_code":"IT","type":"education","lineage":["https://openalex.org/I108290504"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Cima, Lorenzo","raw_affiliation_strings":["University of Pisa"],"raw_orcid":"https://orcid.org/0009-0001-5815-0437","affiliations":[{"raw_affiliation_string":"University of Pisa","institution_ids":["https://openalex.org/I108290504"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056754015","display_name":"Stefano Cresci","orcid":"https://orcid.org/0000-0003-0170-2445"},"institutions":[{"id":"https://openalex.org/I4210130157","display_name":"Institute of Informatics and Telematics","ror":"https://ror.org/02gdcn153","country_code":"IT","type":"facility","lineage":["https://openalex.org/I4210130157","https://openalex.org/I4210155236"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Cresci, Stefano","raw_affiliation_strings":["Institute of Informatics and Telematics"],"raw_orcid":"https://orcid.org/0000-0003-0170-2445","affiliations":[{"raw_affiliation_string":"Institute of Informatics and Telematics","institution_ids":["https://openalex.org/I4210130157"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5136329208","display_name":"Roberto Di Pietro","orcid":"https://orcid.org/0000-0003-1909-0336"},"institutions":[{"id":"https://openalex.org/I71920554","display_name":"King Abdullah University of Science and Technology","ror":"https://ror.org/01q3tbs38","country_code":"SA","type":"education","lineage":["https://openalex.org/I71920554"]}],"countries":["SA"],"is_corresponding":false,"raw_author_name":"Di Pietro, Roberto","raw_affiliation_strings":["King Abdullah University of Science and Technology Department of Computer Science"],"raw_orcid":"https://orcid.org/0000-0003-1909-0336","affiliations":[{"raw_affiliation_string":"King Abdullah University of Science and Technology Department of Computer Science","institution_ids":["https://openalex.org/I71920554"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":null,"topics":[],"keywords":[{"id":"https://openalex.org/keywords/embedding","display_name":"Embedding","score":0.515500009059906},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4413999915122986},{"id":"https://openalex.org/keywords/raw-data","display_name":"Raw data","score":0.40549999475479126},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.32359999418258667},{"id":"https://openalex.org/keywords/lossless-compression","display_name":"Lossless compression","score":0.30219998955726624},{"id":"https://openalex.org/keywords/file-format","display_name":"File format","score":0.28859999775886536}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6870999932289124},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6060000061988831},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5414999723434448},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.515500009059906},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4413999915122986},{"id":"https://openalex.org/C132964779","wikidata":"https://www.wikidata.org/wiki/Q2110223","display_name":"Raw data","level":2,"score":0.40549999475479126},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.32359999418258667},{"id":"https://openalex.org/C81081738","wikidata":"https://www.wikidata.org/wiki/Q55542","display_name":"Lossless compression","level":3,"score":0.30219998955726624},{"id":"https://openalex.org/C97250363","wikidata":"https://www.wikidata.org/wiki/Q235557","display_name":"File format","level":2,"score":0.28859999775886536},{"id":"https://openalex.org/C56288433","wikidata":"https://www.wikidata.org/wiki/Q58673","display_name":"Data manipulation language","level":2,"score":0.2874000072479248},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.27000001072883606},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.25929999351501465},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.25540000200271606}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.5281/zenodo.20256463","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.20256463","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Dataset"}],"best_oa_location":{"id":"doi:10.5281/zenodo.20256463","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.20256463","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Dataset"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320322320","display_name":"King Abdullah University of Science and Technology","ror":"https://ror.org/01q3tbs38"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"SOCRATES-300K:":[0],"Large-Scale":[1],"Hallucination":[2,133],"Detection":[3],"Dataset":[4,49,330,683],"for":[5,26,224,363,418,614],"Language":[6,40,667],"Models":[7],"SOCRATES-300K":[8],"is":[9,149,184,233,253],"a":[10,197,244,457,464],"dataset":[11,52,89,155,188,232,283,316,333,559],"of":[12,38,68,131,158,247,344,396,415,509,529,545,575,630,638,665],"297,795":[13,54],"model":[14,169,443,657],"responses":[15,56,73,160,210,428,583,643],"across":[16,62,129,271,371,501,655],"10":[17,63,356],"open-source":[18],"language":[19,64,346,442],"models":[20,84,357],"with":[21,161,239,255,281,388,513,592,605],"verified":[22,479],"hallucination":[23,28,449,454,576,631,648,653],"labels,":[24],"embeddings,":[25],"comparing":[27],"detection":[29,649],"methods.":[30],"It":[31],"accompanies":[32],"the":[33,69,79,88,91,139,147,168,173,187,192,205,208,215,282,315,345,355,372,378,385,389,397,406,416,441,447,475,502,510,530,535,546,552,606],"paper":[34],"\"A":[35],"Geometric":[36,663],"Analysis":[37,664],"Small-sized":[39,666],"Model":[41,668],"Hallucinations\"":[42],"(Ricco,":[43],"Onofri,":[44],"Cima,":[45],"Cresci,":[46],"Di":[47],"Pietro)":[48],"description":[50],"The":[51,83,153,231,332,380,435,488],"contains":[53,141,334],"text":[55,195,438,512,549],"generated":[57,75,430,439,533,550],"from":[58,349,367,422],"200":[59,70,373],"factual":[60,142,374],"prompts":[61,381,566],"models.":[65],"For":[66],"each":[67,225,364,419],"prompts,":[71],"150":[72,427],"are":[74,90,114,135,211,382,429,484,498],"per":[76,431],"model,":[77,218,347,570],"yielding":[78],"full":[80,154],"297,795-response":[81],"collection.":[82],"employed":[85],"to":[86,277,351,354,369,404,424,446,568,610,625,677],"generate":[87],"following:":[92],"-":[93,95,97,99,101,103,105,107,109,111,561,578,599,618,635,644,651,671],"mistralai/Mistral-7B-v0.1":[94],"google/gemma-2-9b":[96],"google/gemma-2-27b":[98],"Upstage/SOLAR-10.7B-v1.0":[100],"microsoft/phi-4":[102],"Qwen/Qwen2.5-14B":[104],"Qwen/Qwen2.5-32B":[106],"deepseek-ai/deepseek-llm-7b-base":[108],"meta-llama/Llama-3.1-8B":[110],"01-ai/Yi-1.5-9B":[112],"Responses":[113],"all":[115,130,482,496,622],"labelled":[116],"using":[117,214,534,551,585],"Claude":[118,586],"4.5":[119,588],"Sonnet":[120,587],"API":[121,580],"as":[122,164,321,338],"an":[123],"LLM-as-a-judge":[124],"approach,":[125],"ensuring":[126],"consistent":[127,593],"evaluation":[128],"them.":[132],"labels":[134],"binary:":[136],"1":[137,499],"indicates":[138,146],"response":[140,148,183,417,433,445,476,494,511,532,548],"inaccuracies":[143],"(hallucination),":[144],"0":[145,350,368,423,462],"factually":[150],"accurate":[151],"(genuine).":[152],"was":[156],"made":[157],"300,000":[159],"2,205":[162],"tagged":[163],"2,":[165],"indicating":[166,453,473],"then":[167],"does":[170],"not":[171,641],"know":[172],"response.":[174,468],"In":[175],"this":[176,558,679],"version":[177,199,508],"we":[178],"removed":[179],"these":[180],"responses.":[181],"Each":[182],"represented":[185],"in":[186,189,235,243,317,377,384,444],"two":[190,407],"forms:":[191],"original":[193,531],"raw":[194,206],"and":[196,207,229,264,279,289,296,480,518,616,627,640,646,685],"stemmed":[198,209,547],"created":[200],"through":[201],"Porter":[202],"Stemmer.":[203],"Both":[204],"embedded":[212],"independently":[213],"all-MiniLM-L6-v2":[216],"embedding":[217,528,536,544,554],"which":[219],"produces":[220],"384-D":[221,525,541],"dense":[222,526,542],"vectors":[223],"one.":[226],"File":[227],"format":[228,238,252],"loading":[230],"provided":[234],"Apache":[236],"Parquet":[237,293],"lossless":[240],"compression,":[241],"resulting":[242],"file":[245,294,386],"size":[246],"approximately":[248],"974":[249],"MB.":[250],"This":[251],"compatible":[254],"major":[256],"data":[257,287,623],"analysis":[258,273,615,637],"tools":[259],"including":[260],"pandas,":[261],"PyArrow,":[262],"DuckDB,":[263],"SQL":[265],"engines,":[266],"making":[267],"it":[268],"easily":[269],"accessible":[270],"different":[272,656],"platforms.":[274],"Required":[275],"libraries":[276],"load":[278,314],"work":[280],"are:":[284],"pandas":[285,310,320],"(for":[286,292,298],"manipulation":[288],"loading),":[290],"pyarrow":[291,311],"handling),":[295],"numpy":[297],"numerical":[299],"operations).":[300],"These":[301],"can":[302],"be":[303],"installed":[304],"via":[305,589],"pip":[306,308],"with:":[307],"install":[309],"numpy.":[312],"To":[313],"Python:":[318],"import":[319],"pd":[322],"df":[323],"=":[324],"pd.read_parquet('SOCRATES-300K.parquet')":[325],"print(df.shape)":[326],"#":[327],"(297795,":[328],"11)":[329],"schema":[331],"11":[335],"columns":[336],"organized":[337],"follows:":[339],"model_id":[340],"(int):":[341,360,393,412,450,487],"Numerical":[342],"identifier":[343,362],"ranging":[348,366,421],"9,":[352],"corresponding":[353,390],"employed.":[358],"prompt_id":[359,391],"Unique":[361],"prompt,":[365,398,420],"199":[370],"questions":[375],"used":[376,492,676],"study.":[379],"presents":[383],"prompts.xlsx":[387],"year":[392],"Temporal":[394],"variant":[395],"taking":[399],"values":[400],"2020":[401],"or":[402],"2022":[403],"represent":[405],"time":[408],"periods":[409],"considered.":[410],"response_index":[411],"Sequential":[413],"index":[414],"149,":[425],"since":[426],"prompt.":[432,448],"(string):":[434,505],"complete,":[436],"unmodified":[437],"by":[440],"Binary":[451],"label":[452],"status\u20141":[455],"denotes":[456,463],"hallucinated":[458,639,642],"(factually":[459,466],"incorrect)":[460],"response,":[461],"genuine":[465],"correct)":[467],"verification":[469,594],"(bool):":[470],"Boolean":[471],"flag":[472],"whether":[474],"has":[477],"been":[478],"labeled;":[481],"entries":[483,497],"True.":[485],"temperature":[486,490],"sampling":[489],"parameter":[491],"during":[493],"generation;":[495],"(fixed":[500],"dataset).":[503],"stemmed_response":[504],"A":[506,524,540,662],"preprocessed":[507],"tokenization,":[514],"lowercasing,":[515],"stopword":[516],"removal,":[517],"punctuation":[519],"removal":[520],"applied.":[521],"response_embeddings":[522],"(np.array[float]):":[523,539],"vector":[527,543],"model.":[537,555],"stemmed_response_embeddings":[538],"same":[553],"What":[556],"makes":[557],"useful":[560],"Multi-model":[562],"benchmark:":[563],"All":[564,582],"100":[565],"issued":[567],"every":[569],"enabling":[571],"fair":[572],"cross-model":[573],"comparison":[574],"rates.":[577],"Verified":[579],"labels:":[581],"labeled":[584],"Anthropic":[590],"API,":[591],"status":[595],"(no":[596],"unlabeled":[597],"data).":[598],"Pre-computed":[600],"embeddings:":[601],"Response":[602],"embeddings":[603],"shipped":[604],"data,":[607],"no":[608],"need":[609],"recompute;":[611],"immediately":[612],"usable":[613],"evaluation.":[617],"Reproducible":[619],"experiments:":[620],"Includes":[621],"needed":[624],"reproduce":[626],"evaluate":[628],"algorithm":[629],"detection.":[632],"Intended":[633],"uses":[634],"Structural":[636],"Training":[645],"evaluating":[647],"classifiers":[650],"Studying":[652],"rates":[654],"architectures":[658],"Companion":[659],"resources":[660],"Paper:":[661],"Hallucinations":[669],"Text":[670],"generation":[672],"scripts":[673],"(the":[674],"code":[675],"produce":[678],"dataset):":[680],"Socrates-300K":[681],"Licensing":[682],"(images":[684],"metadata):":[686],"Creative":[687],"Commons":[688],"Attribution-NonCommercial-ShareAlike":[689],"4.0":[690],"International":[691],"(CC":[692],"BY-NC-SA":[693],"4.0).":[694]},"counts_by_year":[],"updated_date":"2026-08-29T07:29:34.045763","created_date":"2026-05-18T00:00:00"}