{"id":"https://openalex.org/W4392581042","doi":"https://doi.org/10.1145/3627508.3638322","title":"Enhancing Human Annotation: Leveraging Large Language Models and Efficient Batch Processing","display_name":"Enhancing Human Annotation: Leveraging Large Language Models and Efficient Batch Processing","publication_year":2024,"publication_date":"2024-03-08","ids":{"openalex":"https://openalex.org/W4392581042","doi":"https://doi.org/10.1145/3627508.3638322"},"language":"en","primary_location":{"id":"doi:10.1145/3627508.3638322","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3627508.3638322","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3627508.3638322","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 ACM SIGIR Conference on Human Information Interaction and Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3627508.3638322","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5052168069","display_name":"Oleg Zendel","orcid":"https://orcid.org/0000-0003-1535-0989"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":true,"raw_author_name":"Oleg Zendel","raw_affiliation_strings":["RMIT University, Australia"],"affiliations":[{"raw_affiliation_string":"RMIT University, Australia","institution_ids":["https://openalex.org/I82951845"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070937840","display_name":"J. Shane Culpepper","orcid":"https://orcid.org/0000-0002-1902-9087"},"institutions":[{"id":"https://openalex.org/I165143802","display_name":"The University of Queensland","ror":"https://ror.org/00rqy9422","country_code":"AU","type":"education","lineage":["https://openalex.org/I165143802"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"J. Shane Culpepper","raw_affiliation_strings":["The University of Queensland, Australia"],"affiliations":[{"raw_affiliation_string":"The University of Queensland, Australia","institution_ids":["https://openalex.org/I165143802"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085925326","display_name":"Falk Scholer","orcid":"https://orcid.org/0000-0001-9094-0810"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Falk Scholer","raw_affiliation_strings":["RMIT University, Australia"],"affiliations":[{"raw_affiliation_string":"RMIT University, Australia","institution_ids":["https://openalex.org/I82951845"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5046377370","display_name":"Paul Thomas","orcid":"https://orcid.org/0000-0003-2425-3136"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Paul Thomas","raw_affiliation_strings":["Microsoft, Australia"],"affiliations":[{"raw_affiliation_string":"Microsoft, Australia","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5052168069"],"corresponding_institution_ids":["https://openalex.org/I82951845"],"apc_list":null,"apc_paid":null,"fwci":3.7565,"has_fulltext":false,"cited_by_count":11,"citation_normalized_percentile":{"value":0.93691427,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"340","last_page":"345"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9944999814033508,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.984499990940094,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.766761302947998},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.7037652134895325},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.6824284195899963},{"id":"https://openalex.org/keywords/variety","display_name":"Variety (cybernetics)","score":0.623839795589447},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.6224277019500732},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5472832918167114},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.47841864824295044},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.46039527654647827},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.44074615836143494},{"id":"https://openalex.org/keywords/cognition","display_name":"Cognition","score":0.4387633502483368},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4280101954936981},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3543643355369568},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.33758753538131714},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.08995640277862549}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.766761302947998},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.7037652134895325},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.6824284195899963},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.623839795589447},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.6224277019500732},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5472832918167114},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.47841864824295044},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.46039527654647827},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.44074615836143494},{"id":"https://openalex.org/C169900460","wikidata":"https://www.wikidata.org/wiki/Q2200417","display_name":"Cognition","level":2,"score":0.4387633502483368},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4280101954936981},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3543643355369568},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.33758753538131714},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.08995640277862549},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3627508.3638322","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3627508.3638322","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3627508.3638322","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 ACM SIGIR Conference on Human Information Interaction and Retrieval","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3627508.3638322","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3627508.3638322","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3627508.3638322","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 ACM SIGIR Conference on Human Information Interaction and Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G7479774403","display_name":null,"funder_award_id":"DP190101113","funder_id":"https://openalex.org/F4320323817","funder_display_name":"Universitas Brawijaya"}],"funders":[{"id":"https://openalex.org/F4320323817","display_name":"Universitas Brawijaya","ror":"https://ror.org/01wk3d929"}],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4392581042.pdf"},"referenced_works_count":16,"referenced_works":["https://openalex.org/W1548128919","https://openalex.org/W1983132692","https://openalex.org/W2008285050","https://openalex.org/W2090035194","https://openalex.org/W2782301536","https://openalex.org/W2794333943","https://openalex.org/W2997591727","https://openalex.org/W3094045953","https://openalex.org/W3188970555","https://openalex.org/W4210764005","https://openalex.org/W4284899329","https://openalex.org/W4384107234","https://openalex.org/W4384652592","https://openalex.org/W4384662964","https://openalex.org/W4385688511","https://openalex.org/W4385691145"],"related_works":["https://openalex.org/W2361861616","https://openalex.org/W2263699433","https://openalex.org/W2377979023","https://openalex.org/W2218034408","https://openalex.org/W2392921965","https://openalex.org/W2358755282","https://openalex.org/W2625833328","https://openalex.org/W1533177136","https://openalex.org/W4380994516","https://openalex.org/W3098003361"],"abstract_inverted_index":{"Large":[0],"language":[1,117],"models":[2,118],"(LLMs)":[3],"are":[4,15],"capable":[5],"of":[6,22,57,98,105,115],"assessing":[7],"document":[8],"and":[9,14,69,91,126],"query":[10],"characteristics,":[11],"including":[12],"relevance,":[13],"now":[16],"being":[17],"used":[18],"for":[19,61,119,129],"a":[20,44,62],"variety":[21],"different":[23],"classification":[24],"labeling":[25],"tasks":[26,121],"as":[27,43,80,82],"well.":[28],"This":[29,108],"study":[30,109],"explores":[31],"how":[32],"to":[33,36,52],"use":[34],"LLMs":[35,75],"classify":[37,53],"an":[38],"information":[39],"need,":[40],"often":[41],"represented":[42],"user":[45],"query.":[46],"In":[47],"particular,":[48],"our":[49],"goal":[50],"is":[51],"the":[54,58,96,103,113],"cognitive":[55],"complexity":[56],"search":[59],"task":[60],"given":[63],"\u201cbackstory\u201d.":[64],"Using":[65],"180":[66],"TREC":[67],"topics":[68],"backstories,":[70],"we":[71],"show":[72,88],"that":[73,89],"GPT-based":[74],"agree":[76],"with":[77],"human":[78,84],"experts":[79],"much":[81],"other":[83,130],"experts.":[85],"We":[86],"also":[87],"batching":[90],"ordering":[92],"can":[93],"significantly":[94],"impact":[95],"accuracy":[97],"GPT-3.5,":[99],"but":[100],"rarely":[101],"alter":[102],"quality":[104],"GPT-4":[106],"predictions.":[107],"provides":[110],"insights":[111],"into":[112],"efficacy":[114],"large":[116],"annotation":[120],"normally":[122],"completed":[123],"by":[124],"humans,":[125],"offers":[127],"recommendations":[128],"similar":[131],"applications.":[132]},"counts_by_year":[{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":3}],"updated_date":"2026-04-09T08:11:56.329763","created_date":"2025-10-10T00:00:00"}