{"id":"https://openalex.org/W4414426629","doi":"https://doi.org/10.1287/isre.2023.0426","title":"Quality Control for Crowd Workers and for Language Models: A Framework for Free-Text Response Evaluation with No Ground Truth","display_name":"Quality Control for Crowd Workers and for Language Models: A Framework for Free-Text Response Evaluation with No Ground Truth","publication_year":2025,"publication_date":"2025-09-23","ids":{"openalex":"https://openalex.org/W4414426629","doi":"https://doi.org/10.1287/isre.2023.0426"},"language":"en","primary_location":{"id":"doi:10.1287/isre.2023.0426","is_oa":false,"landing_page_url":"https://doi.org/10.1287/isre.2023.0426","pdf_url":null,"source":{"id":"https://openalex.org/S202812398","display_name":"Information Systems Research","issn_l":"1047-7047","issn":["1047-7047","1526-5536"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310315699","host_organization_name":"Institute for Operations Research and the Management Sciences","host_organization_lineage":["https://openalex.org/P4310315699"],"host_organization_lineage_names":["Institute for Operations Research and the Management Sciences"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information Systems Research","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5070441489","display_name":"Inbal Yahav","orcid":"https://orcid.org/0000-0002-1513-017X"},"institutions":[{"id":"https://openalex.org/I16391192","display_name":"Tel Aviv University","ror":"https://ror.org/04mhzgx49","country_code":"IL","type":"education","lineage":["https://openalex.org/I16391192"]},{"id":"https://openalex.org/I4210141777","display_name":"Academic College of Tel Aviv-Yafo","ror":"https://ror.org/04cg6c004","country_code":"IL","type":"education","lineage":["https://openalex.org/I4210141777"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Inbal Yahav","raw_affiliation_strings":["Coller School of Management, Tel-Aviv University, Tel Aviv-Yafo 6997801, Israel"],"raw_orcid":"https://orcid.org/0000-0002-1513-017X","affiliations":[{"raw_affiliation_string":"Coller School of Management, Tel-Aviv University, Tel Aviv-Yafo 6997801, Israel","institution_ids":["https://openalex.org/I4210141777","https://openalex.org/I16391192"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090512267","display_name":"Anat Goldstein","orcid":"https://orcid.org/0000-0002-1038-762X"},"institutions":[{"id":"https://openalex.org/I52170813","display_name":"Ariel University","ror":"https://ror.org/03nz8qe97","country_code":"IL","type":"education","lineage":["https://openalex.org/I52170813"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Anat Goldstein","raw_affiliation_strings":["Department of Industrial Engineering and Management, Ariel University, Ariel 4070000, Israel"],"raw_orcid":"https://orcid.org/0000-0002-1038-762X","affiliations":[{"raw_affiliation_string":"Department of Industrial Engineering and Management, Ariel University, Ariel 4070000, Israel","institution_ids":["https://openalex.org/I52170813"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040636767","display_name":"Tomer Geva","orcid":"https://orcid.org/0000-0003-4501-3414"},"institutions":[{"id":"https://openalex.org/I16391192","display_name":"Tel Aviv University","ror":"https://ror.org/04mhzgx49","country_code":"IL","type":"education","lineage":["https://openalex.org/I16391192"]},{"id":"https://openalex.org/I4210141777","display_name":"Academic College of Tel Aviv-Yafo","ror":"https://ror.org/04cg6c004","country_code":"IL","type":"education","lineage":["https://openalex.org/I4210141777"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Tomer Geva","raw_affiliation_strings":["Coller School of Management, Tel-Aviv University, Tel Aviv-Yafo 6997801, Israel"],"raw_orcid":"https://orcid.org/0000-0003-4501-3414","affiliations":[{"raw_affiliation_string":"Coller School of Management, Tel-Aviv University, Tel Aviv-Yafo 6997801, Israel","institution_ids":["https://openalex.org/I4210141777","https://openalex.org/I16391192"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Shahar Meir","orcid":null},"institutions":[{"id":"https://openalex.org/I16391192","display_name":"Tel Aviv University","ror":"https://ror.org/04mhzgx49","country_code":"IL","type":"education","lineage":["https://openalex.org/I16391192"]},{"id":"https://openalex.org/I4210141777","display_name":"Academic College of Tel Aviv-Yafo","ror":"https://ror.org/04cg6c004","country_code":"IL","type":"education","lineage":["https://openalex.org/I4210141777"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Shahar Meir","raw_affiliation_strings":["Coller School of Management, Tel-Aviv University, Tel Aviv-Yafo 6997801, Israel"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Coller School of Management, Tel-Aviv University, Tel Aviv-Yafo 6997801, Israel","institution_ids":["https://openalex.org/I4210141777","https://openalex.org/I16391192"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5041060862","display_name":"Onn Shehory","orcid":"https://orcid.org/0000-0001-9594-7819"},"institutions":[{"id":"https://openalex.org/I13955877","display_name":"Bar-Ilan University","ror":"https://ror.org/03kgsv495","country_code":"IL","type":"education","lineage":["https://openalex.org/I13955877"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Onn Shehory","raw_affiliation_strings":["School of Business Administration, Bar-Ilan University, Ramat Gan 5290002, Israel"],"raw_orcid":"https://orcid.org/0000-0001-9594-7819","affiliations":[{"raw_affiliation_string":"School of Business Administration, Bar-Ilan University, Ramat Gan 5290002, Israel","institution_ids":["https://openalex.org/I13955877"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.25763853,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"37","issue":"2","first_page":"782","last_page":"804"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9950000047683716,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.6837999820709229},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.6633999943733215},{"id":"https://openalex.org/keywords/control","display_name":"Control (management)","score":0.5091000199317932},{"id":"https://openalex.org/keywords/accountability","display_name":"Accountability","score":0.49939998984336853},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.482699990272522},{"id":"https://openalex.org/keywords/service","display_name":"Service (business)","score":0.40880000591278076},{"id":"https://openalex.org/keywords/service-quality","display_name":"Service quality","score":0.39910000562667847},{"id":"https://openalex.org/keywords/generative-model","display_name":"Generative model","score":0.3303000032901764}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7724000215530396},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.6837999820709229},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.6633999943733215},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.5091000199317932},{"id":"https://openalex.org/C2776007630","wikidata":"https://www.wikidata.org/wiki/Q2798912","display_name":"Accountability","level":2,"score":0.49939998984336853},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.482699990272522},{"id":"https://openalex.org/C2780378061","wikidata":"https://www.wikidata.org/wiki/Q25351891","display_name":"Service (business)","level":2,"score":0.40880000591278076},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.40380001068115234},{"id":"https://openalex.org/C140781008","wikidata":"https://www.wikidata.org/wiki/Q1221081","display_name":"Service quality","level":3,"score":0.39910000562667847},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.3481000065803528},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.3424000144004822},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.3303000032901764},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.32659998536109924},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.32199999690055847},{"id":"https://openalex.org/C105409693","wikidata":"https://www.wikidata.org/wiki/Q5937824","display_name":"Human intelligence","level":2,"score":0.3019999861717224},{"id":"https://openalex.org/C527821871","wikidata":"https://www.wikidata.org/wiki/Q228502","display_name":"Access control","level":2,"score":0.30160000920295715},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.28850001096725464},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.2874000072479248},{"id":"https://openalex.org/C2780110086","wikidata":"https://www.wikidata.org/wiki/Q161837","display_name":"Public service","level":2,"score":0.2784000039100647},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.27559998631477356},{"id":"https://openalex.org/C105002631","wikidata":"https://www.wikidata.org/wiki/Q4833645","display_name":"Subject-matter expert","level":3,"score":0.27549999952316284},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.273499995470047},{"id":"https://openalex.org/C48798503","wikidata":"https://www.wikidata.org/wiki/Q877546","display_name":"Reputation","level":2,"score":0.26249998807907104},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2606000006198883},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.258899986743927},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.25459998846054077},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.25209999084472656}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1287/isre.2023.0426","is_oa":false,"landing_page_url":"https://doi.org/10.1287/isre.2023.0426","pdf_url":null,"source":{"id":"https://openalex.org/S202812398","display_name":"Information Systems Research","issn_l":"1047-7047","issn":["1047-7047","1526-5536"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310315699","host_organization_name":"Institute for Operations Research and the Management Sciences","host_organization_lineage":["https://openalex.org/P4310315699"],"host_organization_lineage_names":["Institute for Operations Research and the Management Sciences"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information Systems Research","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":27,"referenced_works":["https://openalex.org/W43048615","https://openalex.org/W1520252399","https://openalex.org/W1967082761","https://openalex.org/W2147687736","https://openalex.org/W2148347694","https://openalex.org/W2150612552","https://openalex.org/W2540093921","https://openalex.org/W2787894218","https://openalex.org/W2952173670","https://openalex.org/W2963963993","https://openalex.org/W3027042170","https://openalex.org/W3083423991","https://openalex.org/W3122119771","https://openalex.org/W3124267214","https://openalex.org/W3151685851","https://openalex.org/W3196268181","https://openalex.org/W3201077663","https://openalex.org/W4212863985","https://openalex.org/W4233413206","https://openalex.org/W4238893454","https://openalex.org/W4284680369","https://openalex.org/W4286110465","https://openalex.org/W4312108492","https://openalex.org/W4387359174","https://openalex.org/W4391136507","https://openalex.org/W4398251646","https://openalex.org/W4403429415"],"related_works":[],"abstract_inverted_index":{"As":[0],"businesses":[1],"increasingly":[2],"rely":[3],"on":[4,46],"large":[5],"language":[6],"models":[7],"(LLMs)":[8],"for":[9,104,136,168,180],"tasks":[10],"such":[11],"as":[12],"customer":[13],"service":[14],"and":[15,33,65,130,140,153,159,172,184,189],"information":[16],"retrieval,":[17],"ensuring":[18],"the":[19,41,57,81,85,88,132,162,186],"accuracy":[20],"of":[21,59,87,164],"their":[22],"responses":[23,48,79],"is":[24,30],"a":[25,51,92,123,177],"critical":[26],"challenge.":[27],"Traditional":[28],"verification":[29],"costly,":[31],"slow,":[32],"often":[34],"requires":[35],"scarce":[36],"domain":[37],"experts.":[38],"We":[39],"introduce":[40],"automated":[42,115],"quality":[43,106,163],"evaluation":[44,117],"based":[45],"textual":[47],"(AQER)":[49],"framework,":[50],"novel,":[52],"cost-effective":[53],"method":[54,126],"to":[55,80,90,127,150],"assess":[56],"correctness":[58],"free-text":[60],"answers":[61],"from":[62],"both":[63],"LLMs":[64,135],"human":[66],"workers":[67,166],"without":[68],"needing":[69],"preexisting":[70],"correct":[71,95],"answers.":[72],"AQER":[73,108,121],"works":[74],"by":[75,98],"intelligently":[76],"aggregating":[77],"multiple":[78],"same":[82],"question,":[83],"leveraging":[84],"wisdom":[86],"crowd":[89,165],"create":[91],"reliable":[93],"synthetic":[94],"answer,":[96],"followed":[97],"an":[99],"iterative":[100],"procedure":[101],"that":[102],"accounts":[103],"response":[105,116],"cues.":[107],"obtains":[109],"state-of-the-art":[110],"performance":[111,149,183],"compared":[112],"with":[113,193],"existing":[114],"baselines.":[118],"For":[119],"managers":[120],"offers":[122,176],"scalable,":[124],"data-driven":[125],"(i)":[128],"evaluate":[129],"select":[131],"best":[133],"performing":[134],"specific":[137],"organizational":[138],"needs":[139],"use":[141],"cases,":[142],"(ii)":[143],"continuously":[144],"monitor":[145],"artificial":[146],"intelligence":[147],"(AI)":[148],"ensure":[151],"reliability":[152],"accountability":[154],"across":[155],"different":[156],"model":[157,182],"versions,":[158],"(iii)":[160],"manage":[161],"essential":[167],"high-quality":[169],"AI":[170,197],"training":[171],"validation.":[173],"AQER,":[174],"thus,":[175],"robust":[178],"mechanism":[179],"improving":[181],"mitigating":[185],"significant":[187],"financial":[188],"reputational":[190],"risks":[191],"associated":[192],"deploying":[194],"untrustworthy":[195],"generative":[196],"technologies.":[198]},"counts_by_year":[],"updated_date":"2026-06-27T06:13:17.587685","created_date":"2025-10-10T00:00:00"}