{"id":"https://openalex.org/W4417264530","doi":"https://doi.org/10.1016/j.is.2025.102660","title":"HILTS: Human-LLM collaboration for effective data labeling","display_name":"HILTS: Human-LLM collaboration for effective data labeling","alternate_titles":[],"publication_year":2025,"publication_date":"2025-12-12","ids":{"openalex":"https://openalex.org/W4417264530","doi":"https://doi.org/10.1016/j.is.2025.102660"},"language":"en","primary_location":{"id":"doi:10.1016/j.is.2025.102660","doi":"https://doi.org/10.1016/j.is.2025.102660","is_oa":false,"landing_page_url":"https://doi.org/10.1016/j.is.2025.102660","pdf_url":null,"source":{"id":"https://openalex.org/S193006928","display_name":"Information Systems","issn_l":"0306-4379","issn":["0306-4379","1873-6076"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["ccf-b","cwts-core","dongbi-b","jufo-2","norway-2","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information Systems","raw_type":"journal-article"},"parent_works":[],"lineage":["https://openalex.org/W4417264530"],"child_works_count":0,"type":"article","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5079412598","display_name":"Juliana Silva Barbosa","orcid":"https://orcid.org/0009-0004-4291-5070","observed_orcids":["https://orcid.org/0009-0004-4291-5070"]},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":true,"contributor_roles":[],"raw_author_name":"Juliana Barbosa","raw_affiliation_strings":["New York University, United States of America"],"raw_orcid":"https://orcid.org/0009-0004-4291-5070","affiliations":[{"raw_affiliation_string":"New York University, United States of America","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070728014","display_name":"E. da S. Alencar","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Eduarda Alencar","raw_affiliation_strings":["New York University, United States of America"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"New York University, United States of America","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049651913","display_name":"Grace Fan","orcid":"https://orcid.org/0000-0001-9020-3642","observed_orcids":["https://orcid.org/0000-0001-9020-3642"]},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Grace Fan","raw_affiliation_strings":["New York University, United States of America"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"New York University, United States of America","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101972578","display_name":"A\u00e9cio S. R. Santos","orcid":"https://orcid.org/0000-0002-5124-7770","observed_orcids":["https://orcid.org/0000-0002-5124-7770"]},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"A\u00e9cio Santos","raw_affiliation_strings":["New York University, United States of America"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"New York University, United States of America","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5006773757","display_name":"Juliana Freire","orcid":"https://orcid.org/0000-0003-3915-7075","observed_orcids":["https://orcid.org/0000-0003-3915-7075","https://orcid.org/0009-0009-6393-551X"]},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Juliana Freire","raw_affiliation_strings":["New York University, United States of America"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"New York University, United States of America","institution_ids":["https://openalex.org/I57206974"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5079412598"],"corresponding_institution_ids":["https://openalex.org/I57206974"],"apc_list":{"value":3410,"currency":"USD","value_usd":3410},"apc_paid":null,"fwci":0.5412,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.64554461,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":97},"biblio":{"volume":"138","issue":null,"first_page":"102660","last_page":"102660"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9154999852180481,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9154999852180481,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.06260000169277191,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.01119999960064888,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/human-in-the-loop-machine-learning","display_name":"human-in-the-loop learning","score":0.8320000171661377},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8379999995231628},{"id":"https://openalex.org/keywords/data-annotation","display_name":"data annotation","score":0.746999979019165},{"id":"https://openalex.org/keywords/active-learning-machine-learning","display_name":"active learning (machine learning)","score":0.8190000057220459},{"id":"https://openalex.org/keywords/pseudo-labeling","display_name":"pseudo-labeling","score":0.9150000214576721},{"id":"https://openalex.org/keywords/class-imbalance","display_name":"class imbalance","score":0.8370000123977661},{"id":"https://openalex.org/keywords/annotation-cost-reduction","display_name":"annotation cost reduction","score":0.6340000033378601},{"id":"https://openalex.org/keywords/hilt","display_name":"hilt","score":0.6240000128746033}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8554999828338623},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.7441999912261963},{"id":"https://openalex.org/C177606310","wikidata":"https://www.wikidata.org/wiki/Q5674297","display_name":"Adaptability","level":2,"score":0.6305000185966492},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.5724999904632568},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5723999738693237},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.5583000183105469},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4860999882221222},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.4487000107765198},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.43849998712539673},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.40290001034736633},{"id":"https://openalex.org/C136389625","wikidata":"https://www.wikidata.org/wiki/Q334384","display_name":"Supervised learning","level":3,"score":0.36550000309944153},{"id":"https://openalex.org/C2776145971","wikidata":"https://www.wikidata.org/wiki/Q30673951","display_name":"Labeled data","level":2,"score":0.3626999855041504},{"id":"https://openalex.org/C2777120189","wikidata":"https://www.wikidata.org/wiki/Q780067","display_name":"Triage","level":2,"score":0.361299991607666},{"id":"https://openalex.org/C207685749","wikidata":"https://www.wikidata.org/wiki/Q2088941","display_name":"Domain knowledge","level":2,"score":0.3409000039100647},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3285999894142151},{"id":"https://openalex.org/C20556612","wikidata":"https://www.wikidata.org/wiki/Q4469374","display_name":"Volume (thermodynamics)","level":2,"score":0.3249000012874603},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.3093999922275543},{"id":"https://openalex.org/C77967617","wikidata":"https://www.wikidata.org/wiki/Q4677561","display_name":"Active learning (machine learning)","level":2,"score":0.30079999566078186},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.28380000591278076},{"id":"https://openalex.org/C105002631","wikidata":"https://www.wikidata.org/wiki/Q4833645","display_name":"Subject-matter expert","level":3,"score":0.26350000500679016},{"id":"https://openalex.org/C75553542","wikidata":"https://www.wikidata.org/wiki/Q178161","display_name":"A priori and a posteriori","level":2,"score":0.250900000333786},{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.25040000677108765}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1016/j.is.2025.102660","doi":"https://doi.org/10.1016/j.is.2025.102660","is_oa":false,"landing_page_url":"https://doi.org/10.1016/j.is.2025.102660","pdf_url":null,"source":{"id":"https://openalex.org/S193006928","display_name":"Information Systems","issn_l":"0306-4379","issn":["0306-4379","1873-6076"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["ccf-b","cwts-core","dongbi-b","jufo-2","norway-2","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G2326639171","display_name":"III: Medium: Dataset Search and Ranking for Data Augmentation and Explanation","funder_award_id":"2106888","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G2477307923","display_name":null,"funder_award_id":"HR0011262087","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"},{"id":"https://openalex.org/G4310992995","display_name":"D-ISN/\u200bCollaborative Research: An Interdisciplinary Approach to the Discovery, Analysis, and Disruption of Wildlife Trafficking Networks","funder_award_id":"2146306","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8591816932","display_name":"D-ISN/\u200bCollaborative Research: An Interdisciplinary Approach to the Discovery, Analysis, and Disruption of Wildlife Trafficking Networks","funder_award_id":"2146312","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F2294704616","display_name":"Advanced Research Projects Agency for Health","ror":"https://ror.org/01r532x50"},{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":66,"referenced_works":["https://openalex.org/W1037504319","https://openalex.org/W1972508834","https://openalex.org/W2008989859","https://openalex.org/W2097749765","https://openalex.org/W2148143831","https://openalex.org/W2158941428","https://openalex.org/W2612634114","https://openalex.org/W2613881683","https://openalex.org/W2772832743","https://openalex.org/W2783294918","https://openalex.org/W2889316299","https://openalex.org/W2890297752","https://openalex.org/W2911424454","https://openalex.org/W2960195332","https://openalex.org/W2966389367","https://openalex.org/W2977474077","https://openalex.org/W2984198605","https://openalex.org/W3030163527","https://openalex.org/W3030888939","https://openalex.org/W3040475362","https://openalex.org/W3042958694","https://openalex.org/W3097078376","https://openalex.org/W3113504957","https://openalex.org/W3124212490","https://openalex.org/W3135367836","https://openalex.org/W3157944077","https://openalex.org/W3160285281","https://openalex.org/W3178585493","https://openalex.org/W3209131957","https://openalex.org/W4206377271","https://openalex.org/W4210883322","https://openalex.org/W4226251359","https://openalex.org/W4303437973","https://openalex.org/W4309279743","https://openalex.org/W4317597544","https://openalex.org/W4319296823","https://openalex.org/W4320401884","https://openalex.org/W4321510833","https://openalex.org/W4321610465","https://openalex.org/W4328053680","https://openalex.org/W4366452656","https://openalex.org/W4378417908","https://openalex.org/W4385006113","https://openalex.org/W4385571232","https://openalex.org/W4385782065","https://openalex.org/W4389519254","https://openalex.org/W4389523900","https://openalex.org/W4390608816","https://openalex.org/W4392346169","https://openalex.org/W4392873626","https://openalex.org/W4393971165","https://openalex.org/W4396831993","https://openalex.org/W4396883601","https://openalex.org/W4398157366","https://openalex.org/W4399208169","https://openalex.org/W4401043222","https://openalex.org/W4402671886","https://openalex.org/W4404181466","https://openalex.org/W4404783774","https://openalex.org/W4407727719","https://openalex.org/W4411403123","https://openalex.org/W4412945194","https://openalex.org/W6910351748","https://openalex.org/W6922318549","https://openalex.org/W6941332440","https://openalex.org/W7047884249"],"related_works":["https://openalex.org/W7084239029","https://openalex.org/W4404181466","https://openalex.org/W2990501902","https://openalex.org/W4416237684","https://openalex.org/W4404396960","https://openalex.org/W4392918422","https://openalex.org/W7154085627","https://openalex.org/W7197051442","https://openalex.org/W4408593087","https://openalex.org/W4283792714"],"abstract_inverted_index":null,"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4417264530","counts_by_year":[{"year":2026,"cited_by_count":3}],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2025-12-12T00:00:00"}
