{"id":"https://openalex.org/W3175370130","doi":"https://doi.org/10.1145/3469096.3469872","title":"On minimizing cost in legal document review workflows","display_name":"On minimizing cost in legal document review workflows","publication_year":2021,"publication_date":"2021-08-16","ids":{"openalex":"https://openalex.org/W3175370130","doi":"https://doi.org/10.1145/3469096.3469872","mag":"3175370130"},"language":"en","primary_location":{"id":"doi:10.1145/3469096.3469872","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3469096.3469872","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 21st ACM Symposium on Document Engineering","raw_type":"proceedings-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2106.09866","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Eugene Yang","orcid":null},"institutions":[{"id":"https://openalex.org/I184565670","display_name":"Georgetown University","ror":"https://ror.org/05vzafd60","country_code":"US","type":"education","lineage":["https://openalex.org/I184565670"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Eugene Yang","raw_affiliation_strings":["Georgetown University"],"affiliations":[{"raw_affiliation_string":"Georgetown University","institution_ids":["https://openalex.org/I184565670"]}]},{"author_position":"middle","author":{"id":null,"display_name":"David D. Lewis","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"David D. Lewis","raw_affiliation_strings":["Reveal Data"],"affiliations":[{"raw_affiliation_string":"Reveal Data","institution_ids":[]}]},{"author_position":"last","author":{"id":null,"display_name":"Ophir Frieder","orcid":null},"institutions":[{"id":"https://openalex.org/I184565670","display_name":"Georgetown University","ror":"https://ror.org/05vzafd60","country_code":"US","type":"education","lineage":["https://openalex.org/I184565670"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ophir Frieder","raw_affiliation_strings":["Georgetown University"],"affiliations":[{"raw_affiliation_string":"Georgetown University","institution_ids":["https://openalex.org/I184565670"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I184565670"],"apc_list":null,"apc_paid":null,"fwci":2.2367,"has_fulltext":false,"cited_by_count":16,"citation_normalized_percentile":{"value":0.89912046,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":93,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"10"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11652","display_name":"Imbalanced Data Classification Techniques","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11652","display_name":"Imbalanced Data Classification Techniques","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9962999820709229,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9932000041007996,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.8725000023841858},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6740000247955322},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5479999780654907},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5188999772071838},{"id":"https://openalex.org/keywords/active-learning","display_name":"Active learning (machine learning)","score":0.40139999985694885},{"id":"https://openalex.org/keywords/work","display_name":"Work (physics)","score":0.3720000088214874}],"concepts":[{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.8725000023841858},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6945000290870667},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6740000247955322},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5479999780654907},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5188999772071838},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.41190001368522644},{"id":"https://openalex.org/C77967617","wikidata":"https://www.wikidata.org/wiki/Q4677561","display_name":"Active learning (machine learning)","level":2,"score":0.40139999985694885},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.3720000088214874},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3463999927043915},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.334199994802475},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.32249999046325684},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.3066999912261963},{"id":"https://openalex.org/C143587482","wikidata":"https://www.wikidata.org/wiki/Q1543216","display_name":"Iterative and incremental development","level":2,"score":0.30640000104904175},{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.3003000020980835},{"id":"https://openalex.org/C2781039887","wikidata":"https://www.wikidata.org/wiki/Q1391724","display_name":"Factor (programming language)","level":2,"score":0.2865999937057495},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.27309998869895935},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.2702000141143799},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.26179999113082886},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.2612999975681305},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.2531000077724457}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3469096.3469872","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3469096.3469872","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 21st ACM Symposium on Document Engineering","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2106.09866","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2106.09866","pdf_url":"https://arxiv.org/pdf/2106.09866","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2106.09866","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2106.09866","pdf_url":"https://arxiv.org/pdf/2106.09866","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":26,"referenced_works":["https://openalex.org/W2024932032","https://openalex.org/W2029075138","https://openalex.org/W2039190550","https://openalex.org/W2040066835","https://openalex.org/W2061554433","https://openalex.org/W2065010255","https://openalex.org/W2066344929","https://openalex.org/W2083605078","https://openalex.org/W2123108597","https://openalex.org/W2150102617","https://openalex.org/W2155961949","https://openalex.org/W2214530168","https://openalex.org/W2341384651","https://openalex.org/W2399815506","https://openalex.org/W2533861448","https://openalex.org/W2565695915","https://openalex.org/W2625403256","https://openalex.org/W2776313241","https://openalex.org/W2785836523","https://openalex.org/W2900833639","https://openalex.org/W2955055973","https://openalex.org/W2963961468","https://openalex.org/W3025330860","https://openalex.org/W3089621169","https://openalex.org/W3106480498","https://openalex.org/W3110318438"],"related_works":[],"abstract_inverted_index":{"Technology-assisted":[0],"review":[1,11,19,28,43],"(TAR)":[2],"refers":[3],"to":[4],"human-in-the-loop":[5],"machine":[6],"learning":[7,57,137],"workflows":[8],"for":[9,52,65],"document":[10],"in":[12,85],"legal":[13,23],"discovery":[14],"and":[15,22,42,71,82,97,122,139],"other":[16],"high":[17],"recall":[18],"tasks.":[20],"Attorneys":[21],"technologists":[24],"have":[25],"debated":[26],"whether":[27,39,105],"should":[29,44],"be":[30,45],"a":[31,80,89,106],"single":[32],"iterative":[33],"process":[34],"(one-phase":[35],"TAR":[36,48],"workflows)":[37],"or":[38,108],"model":[40],"training":[41],"separate":[46],"(two-phase":[47],"workflows),":[49],"with":[50],"implications":[51],"the":[53,126],"choice":[54,128],"of":[55,62,72,131,135],"active":[56,136],"algorithm.":[58],"The":[59],"relative":[60,101],"cost":[61,91],"manual":[63],"labeling":[64],"different":[66,73],"purposes":[67],"(training":[68],"vs.":[69,76],"review)":[70],"documents":[74],"(positive":[75],"negative":[77],"examples)":[78],"is":[79],"key":[81],"neglected":[83],"factor":[84],"this":[86],"debate.":[87],"Using":[88],"novel":[90],"dynamics":[92],"analysis,":[93],"we":[94],"show":[95,115],"analytically":[96],"empirically":[98],"that":[99],"these":[100],"costs":[102],"strongly":[103],"impact":[104,125],"one-phase":[107],"two-phase":[109],"workflow":[110,132],"minimizes":[111],"cost.":[112],"We":[113],"also":[114],"how":[116],"category":[117],"prevalence,":[118],"classification":[119],"task":[120],"difficulty,":[121],"collection":[123],"size":[124],"optimal":[127],"not":[129],"only":[130],"type,":[133],"but":[134],"method":[138],"stopping":[140],"point.":[141]},"counts_by_year":[{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":5},{"year":2022,"cited_by_count":3},{"year":2021,"cited_by_count":2}],"updated_date":"2026-03-10T16:38:18.471706","created_date":"2021-07-05T00:00:00"}