{"id":"https://openalex.org/W7155357902","doi":"https://doi.org/10.48550/arxiv.2604.20545","title":"Measuring the Machine: Evaluating Generative AI as Pluralist Sociotechical Systems","display_name":"Measuring the Machine: Evaluating Generative AI as Pluralist Sociotechical Systems","publication_year":2026,"publication_date":"2026-04-22","ids":{"openalex":"https://openalex.org/W7155357902","doi":"https://doi.org/10.48550/arxiv.2604.20545"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.20545","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.20545","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.20545","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5128957589","display_name":"Rebecca L. Johnson","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Johnson, Rebecca L.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5128957589"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.27320000529289246,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.27320000529289246,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11883","display_name":"Embodied and Extended Cognition","score":0.10369999706745148,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T11024","display_name":"Information Systems Theories and Implementation","score":0.046300001442432404,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/sociotechnical-system","display_name":"Sociotechnical system","score":0.9185000061988831},{"id":"https://openalex.org/keywords/reification","display_name":"Reification (Marxism)","score":0.7328000068664551},{"id":"https://openalex.org/keywords/generative-grammar","display_name":"Generative grammar","score":0.7311000227928162},{"id":"https://openalex.org/keywords/meaning","display_name":"Meaning (existential)","score":0.5293999910354614},{"id":"https://openalex.org/keywords/citizen-journalism","display_name":"Citizen journalism","score":0.4018000066280365},{"id":"https://openalex.org/keywords/realism","display_name":"Realism","score":0.37450000643730164},{"id":"https://openalex.org/keywords/formalism","display_name":"Formalism (music)","score":0.3508000075817108},{"id":"https://openalex.org/keywords/value","display_name":"Value (mathematics)","score":0.29319998621940613}],"concepts":[{"id":"https://openalex.org/C127627568","wikidata":"https://www.wikidata.org/wiki/Q1639361","display_name":"Sociotechnical system","level":2,"score":0.9185000061988831},{"id":"https://openalex.org/C201377581","wikidata":"https://www.wikidata.org/wiki/Q863448","display_name":"Reification (Marxism)","level":3,"score":0.7328000068664551},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.7311000227928162},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.6498000025749207},{"id":"https://openalex.org/C2780876879","wikidata":"https://www.wikidata.org/wiki/Q3054749","display_name":"Meaning (existential)","level":2,"score":0.5293999910354614},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.49219998717308044},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4499000012874603},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.4097999930381775},{"id":"https://openalex.org/C203663800","wikidata":"https://www.wikidata.org/wiki/Q848979","display_name":"Citizen journalism","level":2,"score":0.4018000066280365},{"id":"https://openalex.org/C543847140","wikidata":"https://www.wikidata.org/wiki/Q2642826","display_name":"Realism","level":2,"score":0.37450000643730164},{"id":"https://openalex.org/C73301696","wikidata":"https://www.wikidata.org/wiki/Q5469984","display_name":"Formalism (music)","level":3,"score":0.3508000075817108},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.30410000681877136},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.29319998621940613},{"id":"https://openalex.org/C2779561248","wikidata":"https://www.wikidata.org/wiki/Q743861","display_name":"Tacit knowledge","level":2,"score":0.2881999909877777},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.28209999203681946},{"id":"https://openalex.org/C2777655017","wikidata":"https://www.wikidata.org/wiki/Q1501161","display_name":"Toolbox","level":2,"score":0.28040000796318054},{"id":"https://openalex.org/C138673069","wikidata":"https://www.wikidata.org/wiki/Q322229","display_name":"Tracing","level":2,"score":0.2797999978065491},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.2784999907016754},{"id":"https://openalex.org/C47822265","wikidata":"https://www.wikidata.org/wiki/Q854457","display_name":"Complex system","level":2,"score":0.2676999866962433},{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.26579999923706055},{"id":"https://openalex.org/C156325361","wikidata":"https://www.wikidata.org/wiki/Q1152864","display_name":"Grounded theory","level":3,"score":0.26429998874664307},{"id":"https://openalex.org/C2779926162","wikidata":"https://www.wikidata.org/wiki/Q5421173","display_name":"Experimentalism","level":2,"score":0.26249998807907104},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.26030001044273376},{"id":"https://openalex.org/C2775945657","wikidata":"https://www.wikidata.org/wiki/Q381442","display_name":"Structuring","level":2,"score":0.2583000063896179},{"id":"https://openalex.org/C2780440489","wikidata":"https://www.wikidata.org/wiki/Q5227278","display_name":"Data-driven","level":2,"score":0.2554999887943268}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.20545","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.20545","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.20545","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.20545","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"In":[0],"measurement":[1],"theory,":[2],"instruments":[3],"do":[4,23],"not":[5,24,191],"simply":[6],"record":[7],"reality;":[8],"they":[9,27],"help":[10],"constitute":[11],"what":[12,29,45],"is":[13,219],"observed.":[14],"The":[15,194],"same":[16],"holds":[17],"for":[18,98,202],"generative":[19,81,203],"AI":[20,82,227],"evaluation:":[21],"benchmarks":[22,35,199],"just":[25],"measure,":[26],"shape":[28],"models":[30,37],"appear":[31],"to":[32,48,115,182],"be.":[33,49],"Functionalist":[34],"treat":[36],"as":[38,86,131],"isolated":[39],"predictors,":[40],"while":[41],"prescriptive":[42],"approaches":[43],"assess":[44],"systems":[46,228],"ought":[47],"Both":[50],"obscure":[51],"the":[52,63,136,157],"sociotechnical":[53,89,170],"processes":[54],"through":[55,161],"which":[56],"meaning":[57,107],"and":[58,91,103,108,153,169,186,232],"values":[59,118,215],"are":[60,119,188,200,216,229],"enacted,":[61],"risking":[62],"reification":[64],"of":[65,223],"narrow":[66],"cultural":[67],"perspectives":[68],"in":[69,121,145,166,172],"pluralist":[70,88],"contexts.":[71],"This":[72],"thesis":[73,158,195],"advances":[74],"a":[75,87,96,141,221],"descriptive":[76],"alternative.":[77],"It":[78],"argues":[79,196],"that":[80,184,197,211],"must":[83],"be":[84],"evaluated":[85],"system":[90],"develops":[92],"Machine-Society-Human":[93],"(MaSH)":[94],"Loops,":[95],"framework":[97],"tracing":[99],"how":[100,117,226],"models,":[101],"users,":[102],"institutions":[104],"recursively":[105],"co-construct":[106],"values.":[109],"Evaluation":[110,218],"shifts":[111],"from":[112],"judging":[113],"outputs":[114],"examining":[116],"enacted":[120],"interaction.":[122],"Three":[123],"contributions":[124],"follow.":[125],"Conceptually,":[126],"MaSH":[127],"Loops":[128],"reframes":[129],"evaluation":[130,171,187,206],"recursive,":[132],"enactive":[133],"process.":[134],"Methodologically,":[135],"World":[137,146],"Values":[138,147],"Benchmark":[139],"introduces":[140],"distributional":[142],"approach":[143],"grounded":[144],"Survey":[148],"data,":[149],"structured":[150],"prompt":[151],"sets,":[152],"anchor-aware":[154],"scoring.":[155],"Empirically,":[156],"demonstrates":[159],"these":[160],"two":[162],"cases:":[163],"value":[164],"drift":[165],"early":[167],"GPT-3":[168],"real":[173],"estate.":[174],"A":[175],"final":[176],"chapter":[177],"draws":[178],"on":[179],"participatory":[180],"realism":[181],"argue":[183],"prompting":[185],"constitutive":[189],"interventions,":[190],"neutral":[192],"observations.":[193],"static":[198],"insufficient":[201],"AI.":[204],"Responsible":[205],"requires":[207],"pluralist,":[208],"process-oriented":[209],"frameworks":[210],"make":[212],"visible":[213],"whose":[214],"enacted.":[217],"therefore":[220],"site":[222],"governance,":[224],"shaping":[225],"understood,":[230],"deployed,":[231],"trusted.":[233]},"counts_by_year":[],"updated_date":"2026-04-24T06:07:52.864757","created_date":"2026-04-24T00:00:00"}