{"id":"https://openalex.org/W7163648106","doi":"https://doi.org/10.48550/arxiv.2606.06301","title":"More than a Judge: An Empirical Study of Agent-Human Interaction in Crowdsourced Testing Assessment","display_name":"More than a Judge: An Empirical Study of Agent-Human Interaction in Crowdsourced Testing Assessment","publication_year":2026,"publication_date":"2026-06-04","ids":{"openalex":"https://openalex.org/W7163648106","doi":"https://doi.org/10.48550/arxiv.2606.06301"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.06301","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.06301","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.06301","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5137976683","display_name":"Yue Wang (65477)","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Yue","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137950416","display_name":"Yuan Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Yuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101519109","display_name":"Shengcheng Yu","orcid":"https://orcid.org/0000-0003-4640-8637"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yu, Shengcheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138011455","display_name":"Zhenyu Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Zhenyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5137988706","display_name":"Qing Gu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gu, Qing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.7199000120162964,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.7199000120162964,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.07069999724626541,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.03869999945163727,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.6952000260353088},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.6240000128746033},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5733000040054321},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.5562999844551086},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5435000061988831},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.4189000129699707},{"id":"https://openalex.org/keywords/empirical-evidence","display_name":"Empirical evidence","score":0.36489999294281006}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.6952000260353088},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6240000128746033},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5835999846458435},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5733000040054321},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.5562999844551086},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5435000061988831},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4332999885082245},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.4189000129699707},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.38960000872612},{"id":"https://openalex.org/C166052673","wikidata":"https://www.wikidata.org/wiki/Q83021","display_name":"Empirical evidence","level":2,"score":0.36489999294281006},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.3483999967575073},{"id":"https://openalex.org/C191172861","wikidata":"https://www.wikidata.org/wiki/Q7899321","display_name":"Upstream (networking)","level":2,"score":0.3434999883174896},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.328900009393692},{"id":"https://openalex.org/C2776960227","wikidata":"https://www.wikidata.org/wiki/Q2586354","display_name":"Knowledge transfer","level":2,"score":0.3012000024318695},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.29649999737739563},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.2777999937534332},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.26899999380111694},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.26350000500679016},{"id":"https://openalex.org/C150899416","wikidata":"https://www.wikidata.org/wiki/Q1820378","display_name":"Transfer of learning","level":2,"score":0.2524000108242035}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.06301","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.06301","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.06301","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.06301","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Agentic":[0],"AI":[1],"is":[2],"increasingly":[3],"being":[4],"integrated":[5],"into":[6,88,200],"software":[7],"engineering":[8],"workflows.":[9],"In":[10,31],"crowdsourced":[11,100,222],"testing,":[12],"however,":[13],"the":[14,44,97,189,213,220],"large":[15],"volume":[16],"and":[17,36,55,118,164,198,210],"uneven":[18],"quality":[19],"of":[20,99,166],"submitted":[21],"reports":[22,50],"still":[23],"create":[24],"a":[25,38,129,157,172],"substantial":[26],"review":[27],"burden":[28],"for":[29],"developers.":[30],"prior":[32,161],"work,":[33],"we":[34,127],"developed":[35],"validated":[37],"multi-agent":[39],"assessment":[40,68,225],"backbone":[41,48],"based":[42],"on":[43,115,156],"LLM-as-a-Judge":[45],"paradigm.":[46],"That":[47],"assesses":[49],"along":[51],"three":[52,138],"dimensions--textuality,":[53],"adequacy,":[54],"competitiveness--and":[56],"was":[57,191],"shown":[58],"to":[59,171],"align":[60],"well":[61],"with":[62,134],"human":[63,84],"consensus":[64],"while":[65,203],"substantially":[66],"reducing":[67],"effort.":[69],"Yet":[70],"reliable":[71],"automated":[72],"judging":[73],"does":[74],"not":[75,229],"by":[76,179,186],"itself":[77],"show":[78,143],"whether":[79,104],"agent":[80],"outputs":[81],"can":[82,108,227],"improve":[83,109],"work":[85],"when":[86],"embedded":[87],"workflow.":[89],"This":[90],"paper":[91],"studies":[92],"that":[93,144,188,240],"missing":[94],"question":[95],"in":[96,150,196,208,219],"context":[98],"testing.":[101],"We":[102],"investigate":[103],"assessment-derived,":[105],"actionable":[106],"feedback":[107,146,162,190,238],"how":[110],"testers":[111,136],"revise":[112],"reports,":[113,152],"perform":[114],"later":[116,173,201],"tasks,":[117,202],"transfer":[119,170],"reporting":[120],"practices":[121],"across":[122,137],"applications.":[123,140],"To":[124],"do":[125],"so,":[126],"conducted":[128],"controlled":[130],"four-stage":[131],"human-subject":[132],"study":[133,214],"20":[135],"real-world":[139],"The":[141],"results":[142],"agent-generated":[145],"supports":[147],"immediate":[148],"improvements":[149],"revised":[151],"better":[153],"first":[154],"submissions":[155],"new":[158],"task":[159],"after":[160],"exposure,":[163],"evidence":[165,217],"partial":[167],"but":[168,234],"meaningful":[169],"application.":[174],"A":[175],"post-task":[176],"questionnaire":[177],"completed":[178],"17":[180],"participants":[181],"complements":[182],"these":[183],"artifact-based":[184],"findings":[185],"suggesting":[187],"generally":[192],"understandable,":[193],"acted":[194],"upon":[195],"revision,":[197],"carried":[199],"also":[204,235],"revealing":[205],"remaining":[206],"friction":[207],"specificity":[209],"execution.":[211],"Overall,":[212],"provides":[215],"empirical":[216],"that,":[218],"studied":[221],"testing":[223],"setting,":[224],"agents":[226],"serve":[228],"only":[230],"as":[231,236],"post-hoc":[232],"judges":[233],"workflow-integrated":[237],"providers":[239],"support":[241],"upstream":[242],"report-quality":[243],"improvement.":[244]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-06T00:00:00"}