{"id":"https://openalex.org/W3175370130","doi":"https://doi.org/10.1145/3469096.3469872","title":"On minimizing cost in legal document review workflows","display_name":"On minimizing cost in legal document review workflows","publication_year":2021,"publication_date":"2021-08-16","ids":{"openalex":"https://openalex.org/W3175370130","doi":"https://doi.org/10.1145/3469096.3469872","mag":"3175370130"},"language":"en","primary_location":{"id":"doi:10.1145/3469096.3469872","doi":"https://doi.org/10.1145/3469096.3469872","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3469096.3469872","pdf_url":null,"source":{"id":"https://openalex.org/S7407086239","display_name":"ACM Symposium on Document Engineering (DocEng)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 21st ACM Symposium on Document Engineering","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2106.09866","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Eugene Yang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I184565670","display_name":"Georgetown University","ror":"https://ror.org/05vzafd60","country_code":"US","type":"education","lineage":["https://openalex.org/I184565670"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Eugene Yang","raw_affiliation_strings":["Georgetown University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Georgetown University","institution_ids":["https://openalex.org/I184565670"]}]},{"author_position":"middle","author":{"id":null,"display_name":"David D. Lewis","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"David D. Lewis","raw_affiliation_strings":["Reveal Data"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Reveal Data","institution_ids":[]}]},{"author_position":"last","author":{"id":null,"display_name":"Ophir Frieder","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I184565670","display_name":"Georgetown University","ror":"https://ror.org/05vzafd60","country_code":"US","type":"education","lineage":["https://openalex.org/I184565670"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ophir Frieder","raw_affiliation_strings":["Georgetown University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Georgetown University","institution_ids":["https://openalex.org/I184565670"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":29.0847,"has_fulltext":true,"cited_by_count":24,"citation_normalized_percentile":{"value":0.99560658,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"10"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.8604000210762024,"subfield":{"id":"https://openalex.org/subfields/3308","display_name":"Law"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.8604000210762024,"subfield":{"id":"https://openalex.org/subfields/3308","display_name":"Law"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.11800000071525574,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11550","display_name":"Text and Document Classification Technologies","score":0.0031999999191612005,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/technology-assisted-review","display_name":"technology-assisted review","score":0.9940000176429749},{"id":"https://openalex.org/keywords/legal-document-review","display_name":"legal document review","score":0.8690000176429749},{"id":"https://openalex.org/keywords/legal-discovery","display_name":"legal discovery","score":0.7620000243186951},{"id":"https://openalex.org/keywords/cost-minimization","display_name":"cost minimization","score":0.6949999928474426},{"id":"https://openalex.org/keywords/active-learning-machine-learning","display_name":"active learning (machine learning)","score":0.828000009059906},{"id":"https://openalex.org/keywords/human-in-the-loop-machine-learning","display_name":"human-in-the-loop learning","score":0.7739999890327454},{"id":"https://openalex.org/keywords/stopping-points","display_name":"stopping points","score":0.6230000257492065},{"id":"https://openalex.org/keywords/collection-size","display_name":"collection size","score":0.8579999804496765}],"concepts":[{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.8725000023841858},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6945000290870667},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6740000247955322},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5479999780654907},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5188999772071838},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.41190001368522644},{"id":"https://openalex.org/C77967617","wikidata":"https://www.wikidata.org/wiki/Q4677561","display_name":"Active learning (machine learning)","level":2,"score":0.40139999985694885},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.3720000088214874},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3463999927043915},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.334199994802475},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.32249999046325684},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.3066999912261963},{"id":"https://openalex.org/C143587482","wikidata":"https://www.wikidata.org/wiki/Q1543216","display_name":"Iterative and incremental development","level":2,"score":0.30640000104904175},{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.3003000020980835},{"id":"https://openalex.org/C2781039887","wikidata":"https://www.wikidata.org/wiki/Q1391724","display_name":"Factor (programming language)","level":2,"score":0.2865999937057495},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.27309998869895935},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.2702000141143799},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.26179999113082886},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.2612999975681305},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.2531000077724457}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3469096.3469872","doi":"https://doi.org/10.1145/3469096.3469872","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3469096.3469872","pdf_url":null,"source":{"id":"https://openalex.org/S7407086239","display_name":"ACM Symposium on Document Engineering (DocEng)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 21st ACM Symposium on Document Engineering","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2106.09866","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2106.09866","pdf_url":"https://arxiv.org/pdf/2106.09866","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2106.09866","doi":"https://doi.org/10.48550/arxiv.2106.09866","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2106.09866","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article-journal"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2106.09866","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2106.09866","pdf_url":"https://arxiv.org/pdf/2106.09866","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3175370130.pdf","grobid_xml":"https://content.openalex.org/works/W3175370130.grobid-xml"},"referenced_works_count":48,"referenced_works":["https://openalex.org/W157916489","https://openalex.org/W1700671489","https://openalex.org/W1964729842","https://openalex.org/W2024932032","https://openalex.org/W2029075138","https://openalex.org/W2039190550","https://openalex.org/W2040066835","https://openalex.org/W2043566294","https://openalex.org/W2046689939","https://openalex.org/W2061554433","https://openalex.org/W2065010255","https://openalex.org/W2066344929","https://openalex.org/W2083605078","https://openalex.org/W2085989833","https://openalex.org/W2123108597","https://openalex.org/W2126105412","https://openalex.org/W2144642230","https://openalex.org/W2150102617","https://openalex.org/W2155961949","https://openalex.org/W2164547069","https://openalex.org/W2214530168","https://openalex.org/W2341384651","https://openalex.org/W2398442511","https://openalex.org/W2399815506","https://openalex.org/W2504511805","https://openalex.org/W2533861448","https://openalex.org/W2565695915","https://openalex.org/W2625403256","https://openalex.org/W2763389080","https://openalex.org/W2776313241","https://openalex.org/W2785836523","https://openalex.org/W2785937154","https://openalex.org/W2900833639","https://openalex.org/W2903158431","https://openalex.org/W2916445399","https://openalex.org/W2951911250","https://openalex.org/W2955055973","https://openalex.org/W2963961468","https://openalex.org/W3005236729","https://openalex.org/W3013164143","https://openalex.org/W3025330860","https://openalex.org/W3088958837","https://openalex.org/W3089621169","https://openalex.org/W3106480498","https://openalex.org/W3110318438","https://openalex.org/W3195948647","https://openalex.org/W4285719527","https://openalex.org/W7223112381"],"related_works":["https://openalex.org/W3197046229","https://openalex.org/W3175391909","https://openalex.org/W2214530168","https://openalex.org/W2994462009","https://openalex.org/W7052557845","https://openalex.org/W2029075138","https://openalex.org/W2784310254","https://openalex.org/W4386510482","https://openalex.org/W4400725025","https://openalex.org/W4385595313"],"abstract_inverted_index":{"Technology-assisted":[0],"review":[1,11,19,28,43],"(TAR)":[2],"refers":[3],"to":[4],"human-in-the-loop":[5],"machine":[6],"learning":[7,57,137],"workflows":[8],"for":[9,52,65],"document":[10],"in":[12,85],"legal":[13,23],"discovery":[14],"and":[15,22,42,71,82,97,122,139],"other":[16],"high":[17],"recall":[18],"tasks.":[20],"Attorneys":[21],"technologists":[24],"have":[25],"debated":[26],"whether":[27,39,105],"should":[29,44],"be":[30,45],"a":[31,80,89,106],"single":[32],"iterative":[33],"process":[34],"(one-phase":[35],"TAR":[36,48],"workflows)":[37],"or":[38,108],"model":[40],"training":[41],"separate":[46],"(two-phase":[47],"workflows),":[49],"with":[50],"implications":[51],"the":[53,126],"choice":[54,128],"of":[55,62,72,131,135],"active":[56,136],"algorithm.":[58],"The":[59],"relative":[60,101],"cost":[61,91],"manual":[63],"labeling":[64],"different":[66,73],"purposes":[67],"(training":[68],"vs.":[69,76],"review)":[70],"documents":[74],"(positive":[75],"negative":[77],"examples)":[78],"is":[79],"key":[81],"neglected":[83],"factor":[84],"this":[86],"debate.":[87],"Using":[88],"novel":[90],"dynamics":[92],"analysis,":[93],"we":[94],"show":[95,115],"analytically":[96],"empirically":[98],"that":[99],"these":[100],"costs":[102],"strongly":[103],"impact":[104,125],"one-phase":[107],"two-phase":[109],"workflow":[110,132],"minimizes":[111],"cost.":[112],"We":[113],"also":[114],"how":[116],"category":[117],"prevalence,":[118],"classification":[119],"task":[120],"difficulty,":[121],"collection":[123],"size":[124],"optimal":[127],"not":[129],"only":[130],"type,":[133],"but":[134],"method":[138],"stopping":[140],"point.":[141]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":9},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":3}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2021-07-05T00:00:00"}
