{"id":"https://openalex.org/W7225129457","doi":"https://doi.org/10.5281/zenodo.23875158","title":"isitright: Auditing Passing Evals by Changing the World They Run In","display_name":"isitright: Auditing Passing Evals by Changing the World They Run In","publication_year":2026,"publication_date":"2026-11-01","ids":{"openalex":"https://openalex.org/W7225129457","doi":"https://doi.org/10.5281/zenodo.23875158"},"language":"en","primary_location":{"id":"doi:10.5281/zenodo.23875158","doi":"https://doi.org/10.5281/zenodo.23875158","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23875158","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"volume":null,"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.5281/zenodo.23875158","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5031247507","display_name":"Nick Pogrebnyakov","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Pogrebnyakov, Nick","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5031247507"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.5907999873161316,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.5907999873161316,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.05429999902844429,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.029500000178813934,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/agent-evaluation","display_name":"agent evaluation","score":0.7319999933242798},{"id":"https://openalex.org/keywords/global-transformation","display_name":"global transformation","score":0.3019999861717224},{"id":"https://openalex.org/keywords/tool-using-agents","display_name":"tool-using agents","score":0.6669999957084656},{"id":"https://openalex.org/keywords/test-case-generation","display_name":"test case generation","score":0.4740000069141388}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6819000244140625},{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.6658999919891357},{"id":"https://openalex.org/C510870499","wikidata":"https://www.wikidata.org/wiki/Q47607","display_name":"SQL","level":2,"score":0.609000027179718},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5648999810218811},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.5016000270843506},{"id":"https://openalex.org/C2164484","wikidata":"https://www.wikidata.org/wiki/Q5170150","display_name":"Core (optical fiber)","level":2,"score":0.4900999963283539},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.44020000100135803},{"id":"https://openalex.org/C3020493868","wikidata":"https://www.wikidata.org/wiki/Q55631277","display_name":"Real world data","level":2,"score":0.37380000948905945},{"id":"https://openalex.org/C2984286706","wikidata":"https://www.wikidata.org/wiki/Q400708","display_name":"Sql server","level":2,"score":0.3495999872684479},{"id":"https://openalex.org/C58489278","wikidata":"https://www.wikidata.org/wiki/Q1172284","display_name":"Data set","level":2,"score":0.2825999855995178},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.2766000032424927},{"id":"https://openalex.org/C192028432","wikidata":"https://www.wikidata.org/wiki/Q845739","display_name":"Query language","level":2,"score":0.26919999718666077},{"id":"https://openalex.org/C5655090","wikidata":"https://www.wikidata.org/wiki/Q192588","display_name":"Relational database","level":2,"score":0.26330000162124634},{"id":"https://openalex.org/C32833848","wikidata":"https://www.wikidata.org/wiki/Q4115054","display_name":"Extensibility","level":2,"score":0.2554999887943268}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.5281/zenodo.23875158","doi":"https://doi.org/10.5281/zenodo.23875158","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23875158","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"doi:10.5281/zenodo.23875159","doi":"https://doi.org/10.5281/zenodo.23875159","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23875159","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.5281/zenodo.23875158","doi":"https://doi.org/10.5281/zenodo.23875158","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23875158","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7224742511","https://openalex.org/W7221816267","https://openalex.org/W7214913868","https://openalex.org/W7203718958","https://openalex.org/W7222648133","https://openalex.org/W7221675509","https://openalex.org/W7134248038","https://openalex.org/W7214765752","https://openalex.org/W7165644528","https://openalex.org/W7204220566"],"abstract_inverted_index":{"A":[0,131,145],"passing":[1,44],"eval":[2,82],"checks":[3,72],"an":[4,32,37,102,106],"agent":[5,45,107,133,164],"or":[6,59],"query":[7,151],"in":[8,112],"one":[9,11,13,17],"world:":[10],"database,":[12],"set":[14],"of":[15,114,128,142],"bookings,":[16],"clock.":[18],"It":[19,47],"leaves":[20],"open":[21],"how":[22],"it":[23],"behaves":[24],"when":[25],"that":[26,78],"world":[27,51,168],"changes.":[28],"We":[29],"introduce":[30],"isitright,":[31],"open-source":[33],"tool":[34],"built":[35],"on":[36,126,140],"extensible":[38],"framework":[39],"for":[40,49,160],"strengthening":[41],"users'":[42],"own":[43],"evals.":[46],"searches":[48],"valid":[50],"changes":[52],"exposing":[53],"disagreements":[54,152],"with":[55,109],"a":[56,67],"reference":[57,111],"checker,":[58],"oracle.":[60],"Users":[61],"supply":[62],"the":[63,110,170],"environment":[64],"and":[65,75,90,175],"reference;":[66],"shared":[68],"core":[69],"generates":[70],"changes,":[71],"their":[73],"validity":[74],"records":[76],"evidence":[77,176],"can":[79],"become":[80],"new":[81],"cases.":[83],"Domain":[84],"adapters":[85],"support":[86],"fixed":[87],"SQL":[88,146],"queries":[89],"live":[91],"tool-using":[92],"agents,":[93],"including":[94],"agents":[95],"whose":[96],"behavior":[97],"varies":[98],"between":[99],"runs.":[100,179],"In":[101],"airline":[103],"case":[104,147],"study,":[105],"agrees":[108],"115":[113],"116":[115],"judged":[116],"original-world":[117],"runs,":[118],"but":[119],"generated":[120],"cancellation":[121],"worlds":[122],"show":[123],"increased":[124],"disagreement":[125],"eight":[127],"nine":[129],"tasks.":[130],"second":[132],"reaches":[134],"at":[135],"least":[136],"75%":[137],"observed":[138],"agreement":[139],"five":[141],"those":[143],"worlds.":[144],"study":[148],"finds":[149],"14":[150],"satisfying":[153],"database":[154],"structural":[155],"checks.":[156],"These":[157],"are":[158],"findings":[159],"review,":[161],"not":[162],"established":[163],"defects:":[165],"reports":[166],"distinguish":[167],"validity,":[169],"reference's":[171],"assumptions,":[172],"execution":[173],"conditions":[174],"from":[177],"repeated":[178]},"counts_by_year":[],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2026-11-03T00:00:00"}