{"id":"https://openalex.org/W7204197982","doi":"https://doi.org/10.5281/zenodo.22099783","title":"Silent Failure Modes in Free-Tier Graph Database Benchmarking: A Methodology and Four Case Studies","display_name":"Silent Failure Modes in Free-Tier Graph Database Benchmarking: A Methodology and Four Case Studies","publication_year":2026,"publication_date":"2026-08-25","ids":{"openalex":"https://openalex.org/W7204197982","doi":"https://doi.org/10.5281/zenodo.22099783"},"language":"en","primary_location":{"id":"doi:10.5281/zenodo.22099783","doi":"https://doi.org/10.5281/zenodo.22099783","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.22099783","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"volume":null,"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.5281/zenodo.22099783","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5135708895","display_name":"Satvik Mishra","orcid":"https://orcid.org/0009-0002-8778-6845","observed_orcids":["https://orcid.org/0009-0002-8778-6845"]},"institutions":[{"id":"https://openalex.org/I863896202","display_name":"Delhi Technological University","ror":"https://ror.org/01ztcvt22","country_code":"IN","type":"education","lineage":["https://openalex.org/I863896202"]}],"countries":["IN"],"is_corresponding":true,"raw_author_name":"Mishra, Satvik","raw_affiliation_strings":["Delhi Technological University"],"raw_orcid":"https://orcid.org/0009-0002-8778-6845","affiliations":[{"raw_affiliation_string":"Delhi Technological University","institution_ids":["https://openalex.org/I863896202"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5135708895"],"corresponding_institution_ids":["https://openalex.org/I863896202"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10317","display_name":"Advanced Database Systems and Queries","score":0.5360999703407288,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10317","display_name":"Advanced Database Systems and Queries","score":0.5360999703407288,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12292","display_name":"Graph Theory and Algorithms","score":0.3005000054836273,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.0689999982714653,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/social-graphs","display_name":"social graphs","score":0.40299999713897705},{"id":"https://openalex.org/keywords/network-latency","display_name":"network latency","score":0.5770000219345093},{"id":"https://openalex.org/keywords/access-paths","display_name":"access paths","score":0.5590000152587891},{"id":"https://openalex.org/keywords/resource-constraints","display_name":"resource constraints","score":0.5260000228881836},{"id":"https://openalex.org/keywords/distributed-graph-databases","display_name":"distributed graph databases","score":0.5709999799728394},{"id":"https://openalex.org/keywords/reproducibility","display_name":"reproducibility","score":0.640999972820282},{"id":"https://openalex.org/keywords/query-correctness","display_name":"query correctness","score":0.7289999723434448}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7346000075340271},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5929999947547913},{"id":"https://openalex.org/C140745168","wikidata":"https://www.wikidata.org/wiki/Q1210082","display_name":"Tree traversal","level":2,"score":0.5067999958992004},{"id":"https://openalex.org/C183469790","wikidata":"https://www.wikidata.org/wiki/Q333501","display_name":"Crash","level":2,"score":0.48260000348091125},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.47859999537467957},{"id":"https://openalex.org/C79974875","wikidata":"https://www.wikidata.org/wiki/Q483639","display_name":"Cloud computing","level":2,"score":0.4763000011444092},{"id":"https://openalex.org/C41458344","wikidata":"https://www.wikidata.org/wiki/Q732577","display_name":"Publication","level":2,"score":0.4618000090122223},{"id":"https://openalex.org/C176225458","wikidata":"https://www.wikidata.org/wiki/Q595971","display_name":"Graph database","level":3,"score":0.4120999872684479},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.40799999237060547},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.4072999954223633},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.3776000142097473},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3637999892234802},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3619000017642975},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.36039999127388},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.33970001339912415},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.2793000042438507},{"id":"https://openalex.org/C97854310","wikidata":"https://www.wikidata.org/wiki/Q19541","display_name":"Search engine","level":2,"score":0.27309998869895935}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.5281/zenodo.22099783","doi":"https://doi.org/10.5281/zenodo.22099783","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.22099783","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"},{"id":"doi:10.5281/zenodo.22099784","doi":"https://doi.org/10.5281/zenodo.22099784","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.22099784","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.5281/zenodo.22099783","doi":"https://doi.org/10.5281/zenodo.22099783","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.22099783","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Comparative":[0],"benchmarks":[1],"of":[2,70,89,124,155,190,257],"cloud":[3],"database":[4],"services":[5],"are":[6,18,57],"widely":[7,10],"published":[8],"and":[9,61,73,82,109,149,218,239,242],"trusted,":[11],"yet":[12],"the":[13,41,90,234,236,240,246,254,258],"conditions":[14],"under":[15,201],"which":[16,156,179],"they":[17],"run\u2014constrained":[19],"free":[20],"tiers,":[21],"geographically":[22],"distant":[23],"clients,":[24],"heterogeneous":[25],"engine":[26,99,183],"internals\u2014create":[27],"failure":[28,146],"modes":[29,147],"that":[30,37,56,101,165,225,245],"do":[31],"not":[32,45,103,121],"announce":[33],"themselves.":[34],"A":[35],"benchmark":[36,69,134],"is":[38,110,120,253],"silently":[39],"measuring":[40,137],"wrong":[42],"thing":[43],"does":[44],"crash":[46],"or":[47],"produce":[48],"outliers;":[49],"it":[50,139],"produces":[51],"clean,":[52],"plausible,":[53],"well-ordered":[54],"tables":[55],"ready":[58],"to":[59,187,212,249],"publish":[60],"entirely":[62,211],"meaningless.":[63],"This":[64],"work":[65],"presents":[66],"a":[67,97,113,122,128,133,174,206,220,263,272],"reproducible":[68,264],"four":[71,144],"managed":[72],"self-hosted":[74],"graph":[75,163],"databases":[76],"(CognoDB":[77],"Cloud,":[78],"Neo4j":[79],"Aura,":[80],"Memgraph,":[81],"FalkorDB)":[83],"on":[84],"an":[85,213,277],"identical":[86,197],"204,109-relationship":[87],"subgraph":[88],"SNAP":[91],"soc-Pokec":[92],"social":[93],"network,":[94],"together":[95],"with":[96],"fifth":[98],"(NebulaGraph)":[100],"could":[102],"be":[104],"run":[105],"at":[106],"resource":[107,223],"parity":[108],"reported":[111],"as":[112],"documented":[114],"negative":[115],"result.":[116],"The":[117],"primary":[118],"contribution":[119],"ranking":[123],"these":[125,251],"systems":[126],"but":[127],"methodology":[129],"for":[130,276],"detecting":[131],"when":[132],"has":[135],"stopped":[136],"what":[138],"claims":[140],"to.":[141],"We":[142,260],"document":[143],"concrete":[145],"encountered":[148],"corrected":[150],"during":[151],"this":[152],"study,":[153],"each":[154,231],"initially":[157],"produced":[158],"credible-looking":[159],"numbers:":[160],"(1)":[161],"disconnected":[162],"sampling":[164],"renders":[166],"traversal":[167],"latency":[168,176],"meaningless":[169],"while":[170],"appearing":[171],"excellent;":[172],"(2)":[173],"network-dominated":[175],"regime":[177],"in":[178,267],"round-trip":[180],"time":[181,184],"exceeds":[182],"by":[185,271],"up":[186],"two":[188],"orders":[189],"magnitude,":[191],"defeating":[192],"naive":[193],"subtraction-based":[194],"correction;":[195],"(3)":[196],"query":[198],"text":[199],"executing":[200],"non-equivalent":[202],"access":[203],"paths,":[204],"yielding":[205],"285x":[207],"performance":[208],"difference":[209],"attributable":[210],"implicit":[214],"versus":[215],"explicit":[216],"index;":[217],"(4)":[219],"nominally":[221],"neutral":[222],"cap":[224],"structurally":[226],"excludes":[227],"distributed":[228],"architectures.":[229],"For":[230],"we":[232,243],"give":[233],"observation,":[235],"root":[237],"cause,":[238],"correction,":[241],"argue":[244],"machinery":[247],"required":[248],"catch":[250],"failures":[252],"transferable":[255],"output":[256],"work.":[259],"additionally":[261],"report":[262],"query-correctness":[265],"defect":[266],"one":[268],"engine,":[269],"discovered":[270],"data-validation":[273],"check":[274],"written":[275],"unrelated":[278],"purpose.":[279]},"counts_by_year":[],"updated_date":"2026-10-07T09:51:55.459714","created_date":"2026-08-26T00:00:00"}