{"id":"https://openalex.org/W4404058379","doi":"https://doi.org/10.1145/3702639","title":"When Automated Assessment Meets Automated Content Generation: Examining Text Quality in the Era of GPTs","display_name":"When Automated Assessment Meets Automated Content Generation: Examining Text Quality in the Era of GPTs","publication_year":2024,"publication_date":"2024-11-05","ids":{"openalex":"https://openalex.org/W4404058379","doi":"https://doi.org/10.1145/3702639"},"language":"en","primary_location":{"id":"doi:10.1145/3702639","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3702639","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3702639","source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3702639","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5092535966","display_name":"Marialena Bevilacqua","orcid":null},"institutions":[{"id":"https://openalex.org/I107639228","display_name":"University of Notre Dame","ror":"https://ror.org/00mkhxb43","country_code":"US","type":"education","lineage":["https://openalex.org/I107639228"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Marialena Bevilacqua","raw_affiliation_strings":["University of Notre Dame, Notre Dame, Indiana, USA","University of Notre Dame, USA"],"raw_orcid":"https://orcid.org/0009-0001-1641-6428","affiliations":[{"raw_affiliation_string":"University of Notre Dame, Notre Dame, Indiana, USA","institution_ids":["https://openalex.org/I107639228"]},{"raw_affiliation_string":"University of Notre Dame, USA","institution_ids":["https://openalex.org/I107639228"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092964203","display_name":"Kezia Oketch","orcid":"https://orcid.org/0009-0000-1089-2530"},"institutions":[{"id":"https://openalex.org/I107639228","display_name":"University of Notre Dame","ror":"https://ror.org/00mkhxb43","country_code":"US","type":"education","lineage":["https://openalex.org/I107639228"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kezia Oketch","raw_affiliation_strings":["University of Notre Dame, Notre Dame, Indiana, USA","University of Notre Dame, USA"],"raw_orcid":"https://orcid.org/0009-0000-1089-2530","affiliations":[{"raw_affiliation_string":"University of Notre Dame, Notre Dame, Indiana, USA","institution_ids":["https://openalex.org/I107639228"]},{"raw_affiliation_string":"University of Notre Dame, USA","institution_ids":["https://openalex.org/I107639228"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106463942","display_name":"Ruiyang Qin","orcid":"https://orcid.org/0000-0003-0827-2257"},"institutions":[{"id":"https://openalex.org/I107639228","display_name":"University of Notre Dame","ror":"https://ror.org/00mkhxb43","country_code":"US","type":"education","lineage":["https://openalex.org/I107639228"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ruiyang Qin","raw_affiliation_strings":["University of Notre Dame, Notre Dame, Indiana, USA","University of Notre Dame, USA"],"raw_orcid":"https://orcid.org/0000-0003-0827-2257","affiliations":[{"raw_affiliation_string":"University of Notre Dame, Notre Dame, Indiana, USA","institution_ids":["https://openalex.org/I107639228"]},{"raw_affiliation_string":"University of Notre Dame, USA","institution_ids":["https://openalex.org/I107639228"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092964204","display_name":"Will Stamey","orcid":"https://orcid.org/0009-0006-0374-5878"},"institutions":[{"id":"https://openalex.org/I107639228","display_name":"University of Notre Dame","ror":"https://ror.org/00mkhxb43","country_code":"US","type":"education","lineage":["https://openalex.org/I107639228"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Will Stamey","raw_affiliation_strings":["University of Notre Dame, Notre Dame, Indiana, USA","University of Notre Dame, USA"],"raw_orcid":"https://orcid.org/0009-0006-0374-5878","affiliations":[{"raw_affiliation_string":"University of Notre Dame, Notre Dame, Indiana, USA","institution_ids":["https://openalex.org/I107639228"]},{"raw_affiliation_string":"University of Notre Dame, USA","institution_ids":["https://openalex.org/I107639228"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050265041","display_name":"Xinyuan Zhang","orcid":"https://orcid.org/0000-0002-5119-0073"},"institutions":[{"id":"https://openalex.org/I107639228","display_name":"University of Notre Dame","ror":"https://ror.org/00mkhxb43","country_code":"US","type":"education","lineage":["https://openalex.org/I107639228"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xinyuan Zhang","raw_affiliation_strings":["University of Notre Dame, Notre Dame, Indiana, USA","University of Notre Dame, USA"],"raw_orcid":"https://orcid.org/0000-0002-5119-0073","affiliations":[{"raw_affiliation_string":"University of Notre Dame, Notre Dame, Indiana, USA","institution_ids":["https://openalex.org/I107639228"]},{"raw_affiliation_string":"University of Notre Dame, USA","institution_ids":["https://openalex.org/I107639228"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113747196","display_name":"Yi Gan","orcid":"https://orcid.org/0009-0003-0852-0705"},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]},{"id":"https://openalex.org/I2800444561","display_name":"Atlanta Technical College","ror":"https://ror.org/01s3vfp47","country_code":"US","type":"education","lineage":["https://openalex.org/I2800444561"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yi Gan","raw_affiliation_strings":["Georgia Tech, Atlanta, Georgia, USA","Georgia Tech, USA"],"raw_orcid":"https://orcid.org/0009-0003-0852-0705","affiliations":[{"raw_affiliation_string":"Georgia Tech, Atlanta, Georgia, USA","institution_ids":["https://openalex.org/I130701444","https://openalex.org/I2800444561"]},{"raw_affiliation_string":"Georgia Tech, USA","institution_ids":["https://openalex.org/I130701444","https://openalex.org/I2800444561"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037185687","display_name":"Kai Yang","orcid":"https://orcid.org/0000-0002-6371-7741"},"institutions":[{"id":"https://openalex.org/I180726961","display_name":"Shenzhen University","ror":"https://ror.org/01vy4gh70","country_code":"CN","type":"education","lineage":["https://openalex.org/I180726961"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kai Yang","raw_affiliation_strings":["Shenzhen University, Shenzhen, China","Shenzhen University, China"],"raw_orcid":"https://orcid.org/0000-0002-6371-7741","affiliations":[{"raw_affiliation_string":"Shenzhen University, Shenzhen, China","institution_ids":["https://openalex.org/I180726961"]},{"raw_affiliation_string":"Shenzhen University, China","institution_ids":["https://openalex.org/I180726961"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5016943900","display_name":"Ahmed Abbasi","orcid":"https://orcid.org/0000-0001-7698-7794"},"institutions":[{"id":"https://openalex.org/I107639228","display_name":"University of Notre Dame","ror":"https://ror.org/00mkhxb43","country_code":"US","type":"education","lineage":["https://openalex.org/I107639228"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ahmed Abbasi","raw_affiliation_strings":["University of Notre Dame, Notre Dame, Indiana, USA","University of Notre Dame, USA"],"raw_orcid":"https://orcid.org/0000-0001-7698-7794","affiliations":[{"raw_affiliation_string":"University of Notre Dame, Notre Dame, Indiana, USA","institution_ids":["https://openalex.org/I107639228"]},{"raw_affiliation_string":"University of Notre Dame, USA","institution_ids":["https://openalex.org/I107639228"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":6.3293,"has_fulltext":false,"cited_by_count":26,"citation_normalized_percentile":{"value":0.9701807,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":"43","issue":"2","first_page":"1","last_page":"36"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.994700014591217,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.994700014591217,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13523","display_name":"Mathematics, Computing, and Information Processing","score":0.9847000241279602,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9815999865531921,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.564812183380127},{"id":"https://openalex.org/keywords/content","display_name":"Content (measure theory)","score":0.5387109518051147},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.49977946281433105},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.0689312219619751},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.051108211278915405}],"concepts":[{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.564812183380127},{"id":"https://openalex.org/C2778152352","wikidata":"https://www.wikidata.org/wiki/Q5165061","display_name":"Content (measure theory)","level":2,"score":0.5387109518051147},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.49977946281433105},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0689312219619751},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.051108211278915405},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3702639","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3702639","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3702639","source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3702639","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3702639","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3702639","source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.8700000047683716,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4404058379.pdf"},"referenced_works_count":106,"referenced_works":["https://openalex.org/W193524605","https://openalex.org/W1511133876","https://openalex.org/W1527697781","https://openalex.org/W1589394566","https://openalex.org/W1590881479","https://openalex.org/W1832693441","https://openalex.org/W1954968711","https://openalex.org/W2005624335","https://openalex.org/W2007244551","https://openalex.org/W2016278946","https://openalex.org/W2028587098","https://openalex.org/W2031096619","https://openalex.org/W2032165060","https://openalex.org/W2036150701","https://openalex.org/W2039704177","https://openalex.org/W2054745812","https://openalex.org/W2092475127","https://openalex.org/W2110637208","https://openalex.org/W2123442489","https://openalex.org/W2130736933","https://openalex.org/W2137704276","https://openalex.org/W2154541934","https://openalex.org/W2185094885","https://openalex.org/W2250372169","https://openalex.org/W2295598076","https://openalex.org/W2440757793","https://openalex.org/W2465309725","https://openalex.org/W2564237127","https://openalex.org/W2567547739","https://openalex.org/W2599743206","https://openalex.org/W2728810582","https://openalex.org/W2784271235","https://openalex.org/W2799207367","https://openalex.org/W2804545603","https://openalex.org/W2896018557","https://openalex.org/W2905931456","https://openalex.org/W2963830885","https://openalex.org/W2964413085","https://openalex.org/W2965373594","https://openalex.org/W2970702233","https://openalex.org/W2973258416","https://openalex.org/W2981411165","https://openalex.org/W2997195635","https://openalex.org/W3004561114","https://openalex.org/W3006224918","https://openalex.org/W3019295470","https://openalex.org/W3022435406","https://openalex.org/W3034730645","https://openalex.org/W3114208363","https://openalex.org/W3118485687","https://openalex.org/W3146366485","https://openalex.org/W3174822636","https://openalex.org/W3193704761","https://openalex.org/W3201077663","https://openalex.org/W3216314518","https://openalex.org/W4200195738","https://openalex.org/W4205544037","https://openalex.org/W4210468047","https://openalex.org/W4220774147","https://openalex.org/W4223982309","https://openalex.org/W4224212929","https://openalex.org/W4230097545","https://openalex.org/W4245913251","https://openalex.org/W4281611690","https://openalex.org/W4285192297","https://openalex.org/W4287890645","https://openalex.org/W4288104672","https://openalex.org/W4288269198","https://openalex.org/W4288331351","https://openalex.org/W4289523162","https://openalex.org/W4292595691","https://openalex.org/W4297734170","https://openalex.org/W4301912686","https://openalex.org/W4307713853","https://openalex.org/W4309259993","https://openalex.org/W4313479734","https://openalex.org/W4315705277","https://openalex.org/W4318263917","https://openalex.org/W4319301446","https://openalex.org/W4319662928","https://openalex.org/W4320495408","https://openalex.org/W4322718191","https://openalex.org/W4327810158","https://openalex.org/W4353015365","https://openalex.org/W4361806892","https://openalex.org/W4362598574","https://openalex.org/W4365799947","https://openalex.org/W4366403195","https://openalex.org/W4366999185","https://openalex.org/W4368755092","https://openalex.org/W4379792729","https://openalex.org/W4384071683","https://openalex.org/W4385571576","https://openalex.org/W4386641961","https://openalex.org/W4386746103","https://openalex.org/W4388182168","https://openalex.org/W4388400861","https://openalex.org/W4388843373","https://openalex.org/W4389437528","https://openalex.org/W4390691951","https://openalex.org/W4391136348","https://openalex.org/W4398251646","https://openalex.org/W4402329213","https://openalex.org/W6630710947","https://openalex.org/W7052923770","https://openalex.org/W7077157121"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"The":[0],"use":[1],"of":[2,20,34,43,75,109,111,136,247],"machine":[3],"learning":[4,187],"(ML)":[5],"models":[6,50,67,147,191],"to":[7,157,172,181,218,230,251],"assess":[8,60,72],"and":[9,29,31,45,97,100,115,132,143,159,168,188,205,236],"score":[10,151,173,192],"textual":[11],"data":[12],"has":[13],"become":[14],"increasingly":[15],"pervasive":[16],"in":[17,63,223,232],"an":[18,87],"array":[19],"contexts":[21],"including":[22],"natural":[23],"language":[24,146],"processing,":[25],"information":[26],"retrieval,":[27],"search":[28],"recommendation,":[30],"credibility":[32],"assessment":[33,121],"online":[35],"content.":[36],"A":[37,123],"significant":[38],"disruption":[39],"at":[40],"the":[41,61,73,107,116,203],"intersection":[42],"ML":[44,94,117,161,190],"text":[46,175,194,241,248],"are":[47,208],"text-generating":[48],"large-language":[49],"(LLMs)":[51],"such":[52],"as":[53,155],"generative":[54,255],"pre-trained":[55],"transformers":[56],"(GPTs).":[57],"We":[58],"empirically":[59],"differences":[62],"how":[64],"ML-based":[65],"scoring":[66,93,246],"trained":[68],"on":[69,178,211],"human":[70,152,193,212],"content":[71,76],"quality":[74,154],"generated":[77],"by":[78,254],"humans":[79],"versus":[80],"GPTs.":[81],"To":[82],"do":[83],"so,":[84],"we":[85,164],"propose":[86],"analysis":[88,139,198],"framework":[89,235],"that":[90,104,128,141,166,200],"encompasses":[91,129],"essay":[92,153],"models,":[95],"human-":[96],"ML-generated":[98],"essays,":[99],"a":[101],"statistical":[102],"model":[103,118],"parsimoniously":[105],"considers":[106],"impact":[108],"type":[110],"respondent,":[112],"prompt":[113],"genre,":[114],"used":[119],"for":[120,240],"model.":[122],"rich":[124],"testbed":[125],"is":[126,249],"utilized":[127],"18,460":[130],"human-generated":[131],"GPT-based":[133],"essays.":[134],"Results":[135],"our":[137],"benchmark":[138],"reveal":[140],"LLMs":[142,167,204],"transformer":[144,169,206],"pretrained":[145],"(PLMs)":[148],"more":[149,215],"accurately":[150],"compared":[156],"CNN/RNN":[158],"feature-based":[160,189],"methods.":[162],"Interestingly,":[163],"find":[165],"PLMs":[170,207],"tend":[171],"GPT-generated":[174,224],"10\u201320%":[176],"higher":[177],"average,":[179],"relative":[180],"human-authored":[182],"documents.":[183],"Conversely,":[184],"traditional":[185],"deep":[186],"considerably":[195],"higher.":[196],"Further":[197],"reveals":[199],"even":[201],"though":[202],"exclusively":[209],"fine-tuned":[210],"text,":[213,225],"they":[214],"prominently":[216],"attend":[217],"certain":[219],"tokens":[220],"appearing":[221],"only":[222],"possibly":[226],"(in":[227],"part)":[228],"due":[229],"familiarity/overlap":[231],"pre-training.":[233],"Our":[234],"results":[237],"have":[238],"implications":[239],"classification":[242],"settings":[243],"where":[244],"automated":[245],"likely":[250],"be":[252],"disrupted":[253],"AI.":[256]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":17},{"year":2024,"cited_by_count":3}],"updated_date":"2026-07-22T07:51:19.307946","created_date":"2025-10-10T00:00:00"}