{"id":"https://openalex.org/W2060480356","doi":"https://doi.org/10.1145/2348283.2348530","title":"Crowdsourcing for search evaluation and social-algorithmic search","display_name":"Crowdsourcing for search evaluation and social-algorithmic search","publication_year":2012,"publication_date":"2012-08-12","ids":{"openalex":"https://openalex.org/W2060480356","doi":"https://doi.org/10.1145/2348283.2348530","mag":"2060480356"},"language":"en","primary_location":{"id":"doi:10.1145/2348283.2348530","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2348283.2348530","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 35th international ACM SIGIR conference on Research and development in information retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5023397430","display_name":"Matthew Lease","orcid":"https://orcid.org/0000-0002-0056-2834"},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Matthew Lease","raw_affiliation_strings":["University of Texas at Austin, Austin, TX, USA","University of Texas at Austin, Austin, TX USA"],"affiliations":[{"raw_affiliation_string":"University of Texas at Austin, Austin, TX, USA","institution_ids":["https://openalex.org/I86519309"]},{"raw_affiliation_string":"University of Texas at Austin, Austin, TX USA","institution_ids":["https://openalex.org/I86519309"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5051420635","display_name":"Omar Alonso","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Omar Alonso","raw_affiliation_strings":["Microsoft, Mountain View, CA, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft, Mountain View, CA, USA","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5023397430"],"corresponding_institution_ids":["https://openalex.org/I86519309"],"apc_list":null,"apc_paid":null,"fwci":1.5105,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.88068901,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"1180","last_page":"1180"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9772999882698059,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.960099995136261,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9758126735687256},{"id":"https://openalex.org/keywords/crowds","display_name":"Crowds","score":0.8061004877090454},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7448559403419495},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.6642670631408691},{"id":"https://openalex.org/keywords/crowdsourcing-software-development","display_name":"Crowdsourcing software development","score":0.6301529407501221},{"id":"https://openalex.org/keywords/automation","display_name":"Automation","score":0.5652593374252319},{"id":"https://openalex.org/keywords/the-internet","display_name":"The Internet","score":0.5365310907363892},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.4822436273097992},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.3771066665649414},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.1543172001838684},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.13413330912590027},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.11326825618743896}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9758126735687256},{"id":"https://openalex.org/C2777852691","wikidata":"https://www.wikidata.org/wiki/Q13430821","display_name":"Crowds","level":2,"score":0.8061004877090454},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7448559403419495},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.6642670631408691},{"id":"https://openalex.org/C32810761","wikidata":"https://www.wikidata.org/wiki/Q17082146","display_name":"Crowdsourcing software development","level":5,"score":0.6301529407501221},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.5652593374252319},{"id":"https://openalex.org/C110875604","wikidata":"https://www.wikidata.org/wiki/Q75","display_name":"The Internet","level":2,"score":0.5365310907363892},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.4822436273097992},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.3771066665649414},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.1543172001838684},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.13413330912590027},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.11326825618743896},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C149091818","wikidata":"https://www.wikidata.org/wiki/Q2429814","display_name":"Software system","level":3,"score":0.0},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.0},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2348283.2348530","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2348283.2348530","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 35th international ACM SIGIR conference on Research and development in information retrieval","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9","score":0.41999998688697815}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2512762878","https://openalex.org/W162737300","https://openalex.org/W2520748299","https://openalex.org/W2033745466","https://openalex.org/W2520160102","https://openalex.org/W2046555014","https://openalex.org/W3131060711","https://openalex.org/W2548342535","https://openalex.org/W3125008079","https://openalex.org/W2287983667"],"abstract_inverted_index":{"The":[0],"first":[1],"computers":[2],"were":[3],"people.":[4],"Today,":[5],"Internet-based":[6],"access":[7],"to":[8,15,36],"24/7":[9],"online":[10],"human":[11,21,69,89,104],"crowds":[12],"has":[13],"led":[14],"a":[16,33],"renaissance":[17],"of":[18,26,88,109,117],"research":[19,37,119],"in":[20,78],"computation":[22,70,90,105],"and":[23,38,54,60,86,91,97,115,120,131],"the":[24,74],"advent":[25],"crowdsourcing.":[27],"These":[28],"new":[29],"opportunities":[30,85],"have":[31],"brought":[32],"disruptive":[34],"shift":[35],"practice":[39],"for":[40,52,94],"how":[41],"we":[42,66],"build":[43],"intelligent":[44],"systems":[45,75],"today.":[46],"Not":[47],"only":[48],"can":[49],"labeled":[50],"data":[51],"training":[53],"evaluation":[55,96],"be":[56],"collected":[57],"faster,":[58],"cheaper,":[59],"easier":[61],"than":[62],"ever":[63],"before,":[64],"but":[65],"now":[67,126],"see":[68],"being":[71],"integrated":[72],"into":[73],"themselves,":[76],"operating":[77],"concert":[79],"with":[80,106],"automation.":[81],"This":[82],"tutorial":[83],"introduces":[84],"challenges":[87],"crowdsourcing,":[92],"particularly":[93],"search":[95,100],"developing":[98],"hybrid":[99],"solutions":[101],"that":[102],"integrate":[103],"traditional":[107],"forms":[108],"automated":[110],"search.":[111],"We":[112],"review":[113],"methodology":[114],"findings":[116],"recent":[118],"survey":[121],"current":[122],"generation":[123],"crowdsourcing":[124],"platforms":[125],"available,":[127],"analyzing":[128],"methods,":[129],"potential,":[130],"limitations":[132],"across":[133],"platforms.":[134]},"counts_by_year":[{"year":2019,"cited_by_count":2},{"year":2016,"cited_by_count":2},{"year":2014,"cited_by_count":2},{"year":2013,"cited_by_count":1}],"updated_date":"2026-03-10T16:38:18.471706","created_date":"2025-10-10T00:00:00"}