{"id":"https://openalex.org/W2343442396","doi":"https://doi.org/10.1145/2870649","title":"Crowdsourcing Human Annotation on Web Page Structure","display_name":"Crowdsourcing Human Annotation on Web Page Structure","publication_year":2016,"publication_date":"2016-04-25","ids":{"openalex":"https://openalex.org/W2343442396","doi":"https://doi.org/10.1145/2870649","mag":"2343442396"},"language":"en","primary_location":{"id":"doi:10.1145/2870649","is_oa":true,"landing_page_url":"https://doi.org/10.1145/2870649","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=2870649&type=pdf","source":{"id":"https://openalex.org/S2492086750","display_name":"ACM Transactions on Intelligent Systems and Technology","issn_l":"2157-6904","issn":["2157-6904","2157-6912"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Intelligent Systems and Technology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"http://dl.acm.org/ft_gateway.cfm?id=2870649&type=pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5071179467","display_name":"Shuguang Han","orcid":"https://orcid.org/0000-0003-1416-6960"},"institutions":[{"id":"https://openalex.org/I170201317","display_name":"University of Pittsburgh","ror":"https://ror.org/01an3r305","country_code":"US","type":"education","lineage":["https://openalex.org/I170201317"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shuguang Han","raw_affiliation_strings":["University of Pittsburgh, Pittsburgh, PA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Pittsburgh, Pittsburgh, PA","institution_ids":["https://openalex.org/I170201317"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058646158","display_name":"Peng Dai","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Peng Dai","raw_affiliation_strings":["Google"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076601570","display_name":"Praveen Paritosh","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Praveen Paritosh","raw_affiliation_strings":["Google"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5011028347","display_name":"David Huynh","orcid":"https://orcid.org/0000-0003-1491-5829"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"David Huynh","raw_affiliation_strings":["Google"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google","institution_ids":["https://openalex.org/I1291425158"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":4.9553,"has_fulltext":true,"cited_by_count":17,"citation_normalized_percentile":{"value":0.95133001,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":"7","issue":"4","first_page":"1","last_page":"25"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9947999715805054,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.8762122392654419},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8740377426147461},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.5780725479125977},{"id":"https://openalex.org/keywords/web-page","display_name":"Web page","score":0.5678006410598755},{"id":"https://openalex.org/keywords/replicate","display_name":"Replicate","score":0.5326414108276367},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5184923410415649},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.5134397745132446},{"id":"https://openalex.org/keywords/precision-and-recall","display_name":"Precision and recall","score":0.49799633026123047},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.4967404007911682},{"id":"https://openalex.org/keywords/parsing","display_name":"Parsing","score":0.4948353171348572},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.47489041090011597},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.4290814697742462},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.39525386691093445},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.35983961820602417},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.34395843744277954}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8762122392654419},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8740377426147461},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.5780725479125977},{"id":"https://openalex.org/C21959979","wikidata":"https://www.wikidata.org/wiki/Q36774","display_name":"Web page","level":2,"score":0.5678006410598755},{"id":"https://openalex.org/C2781162219","wikidata":"https://www.wikidata.org/wiki/Q26250693","display_name":"Replicate","level":2,"score":0.5326414108276367},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5184923410415649},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5134397745132446},{"id":"https://openalex.org/C81669768","wikidata":"https://www.wikidata.org/wiki/Q2359161","display_name":"Precision and recall","level":2,"score":0.49799633026123047},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4967404007911682},{"id":"https://openalex.org/C186644900","wikidata":"https://www.wikidata.org/wiki/Q194152","display_name":"Parsing","level":2,"score":0.4948353171348572},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.47489041090011597},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.4290814697742462},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.39525386691093445},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.35983961820602417},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.34395843744277954},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2870649","is_oa":true,"landing_page_url":"https://doi.org/10.1145/2870649","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=2870649&type=pdf","source":{"id":"https://openalex.org/S2492086750","display_name":"ACM Transactions on Intelligent Systems and Technology","issn_l":"2157-6904","issn":["2157-6904","2157-6912"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Intelligent Systems and Technology","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/2870649","is_oa":true,"landing_page_url":"https://doi.org/10.1145/2870649","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=2870649&type=pdf","source":{"id":"https://openalex.org/S2492086750","display_name":"ACM Transactions on Intelligent Systems and Technology","issn_l":"2157-6904","issn":["2157-6904","2157-6912"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Intelligent Systems and Technology","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2343442396.pdf","grobid_xml":"https://content.openalex.org/works/W2343442396.grobid-xml"},"referenced_works_count":31,"referenced_works":["https://openalex.org/W8550301","https://openalex.org/W9014458","https://openalex.org/W82775311","https://openalex.org/W1493490255","https://openalex.org/W1570705485","https://openalex.org/W1597431254","https://openalex.org/W2002982773","https://openalex.org/W2016210396","https://openalex.org/W2043382629","https://openalex.org/W2064971756","https://openalex.org/W2075851722","https://openalex.org/W2078429476","https://openalex.org/W2083563309","https://openalex.org/W2098865355","https://openalex.org/W2101409192","https://openalex.org/W2104749423","https://openalex.org/W2109137581","https://openalex.org/W2110764733","https://openalex.org/W2120101509","https://openalex.org/W2120889650","https://openalex.org/W2124994029","https://openalex.org/W2127008633","https://openalex.org/W2134150392","https://openalex.org/W2138405339","https://openalex.org/W2143280940","https://openalex.org/W2151401338","https://openalex.org/W2152749438","https://openalex.org/W2161494021","https://openalex.org/W2166237624","https://openalex.org/W2167913131","https://openalex.org/W4256361765"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W4384486036","https://openalex.org/W135177976","https://openalex.org/W1503094549","https://openalex.org/W4254851101","https://openalex.org/W2337920774","https://openalex.org/W2886410948","https://openalex.org/W3171007296","https://openalex.org/W4287688258","https://openalex.org/W3049211950"],"abstract_inverted_index":{"Parsing":[0],"the":[1,89,166,232,236],"semantic":[2],"structure":[3],"of":[4,12,40,144,168],"a":[5,9,126,141,156,175],"web":[6,13,136,161,184],"page":[7,162],"is":[8,228,240],"key":[10],"component":[11],"information":[14,137,259,263],"extraction.":[15,138],"Successful":[16],"extraction":[17],"algorithms":[18],"usually":[19],"require":[20,50],"large-scale":[21,42],"training":[22,43],"and":[23,73,103,235,260],"evaluation":[24],"datasets,":[25],"which":[26],"are":[27,97,110,201],"difficult":[28],"to":[29,35,65,132,231,264],"acquire.":[30],"Recently,":[31,101],"crowdsourcing":[32,119,128,205],"has":[33],"proven":[34],"be":[36],"an":[37],"effective":[38],"method":[39,210],"collecting":[41],"data":[44],"in":[45,68,88,117,214],"domains":[46],"that":[47,107],"do":[48],"not":[49],"much":[51],"domain":[52],"knowledge.":[53],"For":[54],"more":[55,86,241],"complex":[56],"domains,":[57],"researchers":[58],"have":[59,105],"proposed":[60],"sophisticated":[61],"quality":[62,116,154,169,250],"control":[63,170],"mechanisms":[64],"replicate":[66],"tasks":[67],"parallel":[69],"or":[70],"sequential":[71],"ways":[72],"then":[74],"aggregate":[75],"responses":[76],"from":[77,186],"multiple":[78],"workers.":[79],"Conventional":[80],"annotation":[81,115,153],"integration":[82],"methods":[83,213],"often":[84],"put":[85],"trust":[87],"workers":[90,180],"with":[91,114,220],"high":[92],"historical":[93,261],"performance;":[94],"thus,":[95],"they":[96],"called":[98,130],"performance-based":[99,212],"methods.":[100],"Rzeszotarski":[102],"Kittur":[104],"demonstrated":[106],"behavioral":[108,145,172,199,226,258],"features":[109,146,173,200,227],"also":[111],"highly":[112],"correlated":[113],"several":[118,195],"applications.":[120],"In":[121,190,223],"this":[122],"article,":[123],"we":[124,193],"present":[125],"new":[127],"system,":[129],"Wernicke,":[131],"provide":[133],"annotations":[134],"for":[135,155,204,243,248],"Wernicke":[139],"collects":[140],"wide":[142],"set":[143],"and,":[147],"based":[148],"on":[149],"these":[150],"features,":[151],"predicts":[152],"challenging":[157],"task":[158],"domain:":[159],"annotating":[160],"structure.":[163],"We":[164],"evaluate":[165],"effectiveness":[167],"using":[171,225],"through":[174],"case":[176],"study":[177],"where":[178],"32":[179],"annotate":[181],"200":[182],"Q&A":[183],"pages":[185],"five":[187],"popular":[188],"websites.":[189],"doing":[191],"so,":[192],"discover":[194],"things:":[196],"(1)":[197],"Many":[198],"significant":[202],"predictors":[203],"quality.":[206],"(2)":[207],"The":[208],"behavioral-feature-based":[209],"outperforms":[211],"recall":[215,245],"prediction,":[216],"while":[217],"performing":[218],"equally":[219],"precision":[221,247],"prediction.":[222],"addition,":[224],"less":[229],"vulnerable":[230],"cold-start":[233],"problem,":[234],"corresponding":[237],"prediction":[238,267],"model":[239],"generalizable":[242],"predicting":[244],"than":[246],"cross-website":[249],"analysis.":[251],"(3)":[252],"One":[253],"can":[254],"effectively":[255],"combine":[256],"workers\u2019":[257],"performance":[262],"further":[265],"reduce":[266],"errors.":[268]},"counts_by_year":[{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":3},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":5},{"year":2018,"cited_by_count":3},{"year":2017,"cited_by_count":1}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}