{"id":"https://openalex.org/W2515979697","doi":"https://doi.org/10.1145/2970398.2970427","title":"Retrievability in API-Based \"Evaluation as a Service\"","display_name":"Retrievability in API-Based \"Evaluation as a Service\"","publication_year":2016,"publication_date":"2016-09-09","ids":{"openalex":"https://openalex.org/W2515979697","doi":"https://doi.org/10.1145/2970398.2970427","mag":"2515979697"},"language":"en","primary_location":{"id":"doi:10.1145/2970398.2970427","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2970398.2970427","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2016 ACM International Conference on the Theory of Information Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5062553068","display_name":"Jiaul H. Paik","orcid":"https://orcid.org/0000-0003-1550-3586"},"institutions":[{"id":"https://openalex.org/I145894827","display_name":"Indian Institute of Technology Kharagpur","ror":"https://ror.org/03w5sq511","country_code":"IN","type":"education","lineage":["https://openalex.org/I145894827"]}],"countries":["IN"],"is_corresponding":true,"raw_author_name":"Jiaul H. Paik","raw_affiliation_strings":["IIT Kharagpur, Kharagpur, India"],"affiliations":[{"raw_affiliation_string":"IIT Kharagpur, Kharagpur, India","institution_ids":["https://openalex.org/I145894827"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5082997975","display_name":"Jimmy Lin","orcid":"https://orcid.org/0000-0002-0661-7189"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Jimmy Lin","raw_affiliation_strings":["University of Waterloo, Waterloo, ON, Canada"],"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, ON, Canada","institution_ids":["https://openalex.org/I151746483"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5062553068"],"corresponding_institution_ids":["https://openalex.org/I145894827"],"apc_list":null,"apc_paid":null,"fwci":1.9904,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.89786203,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"91","last_page":"94"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13976","display_name":"Web visibility and informetrics","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13976","display_name":"Web visibility and informetrics","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.991100013256073,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9573000073432922,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/retrievability","display_name":"Retrievability","score":0.8165978193283081},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7783100605010986},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6532053351402283},{"id":"https://openalex.org/keywords/raw-data","display_name":"Raw data","score":0.5662129521369934},{"id":"https://openalex.org/keywords/service","display_name":"Service (business)","score":0.5354818105697632},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.47142380475997925},{"id":"https://openalex.org/keywords/data-collection","display_name":"Data collection","score":0.4579355716705322},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.4462076723575592},{"id":"https://openalex.org/keywords/application-programming-interface","display_name":"Application programming interface","score":0.44402793049812317},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.4188953936100006},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.10301023721694946},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.08599674701690674},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.07083266973495483},{"id":"https://openalex.org/keywords/document-retrieval","display_name":"Document retrieval","score":0.06999027729034424}],"concepts":[{"id":"https://openalex.org/C191072391","wikidata":"https://www.wikidata.org/wiki/Q17043235","display_name":"Retrievability","level":3,"score":0.8165978193283081},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7783100605010986},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6532053351402283},{"id":"https://openalex.org/C132964779","wikidata":"https://www.wikidata.org/wiki/Q2110223","display_name":"Raw data","level":2,"score":0.5662129521369934},{"id":"https://openalex.org/C2780378061","wikidata":"https://www.wikidata.org/wiki/Q25351891","display_name":"Service (business)","level":2,"score":0.5354818105697632},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.47142380475997925},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.4579355716705322},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.4462076723575592},{"id":"https://openalex.org/C99613125","wikidata":"https://www.wikidata.org/wiki/Q165194","display_name":"Application programming interface","level":2,"score":0.44402793049812317},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4188953936100006},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.10301023721694946},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.08599674701690674},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.07083266973495483},{"id":"https://openalex.org/C161156560","wikidata":"https://www.wikidata.org/wiki/Q1638872","display_name":"Document retrieval","level":2,"score":0.06999027729034424},{"id":"https://openalex.org/C136264566","wikidata":"https://www.wikidata.org/wiki/Q159810","display_name":"Economy","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2970398.2970427","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2970398.2970427","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2016 ACM International Conference on the Theory of Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G6894796833","display_name":null,"funder_award_id":"IIS-1218043 and CNS-1405688","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":10,"referenced_works":["https://openalex.org/W789651694","https://openalex.org/W1554394657","https://openalex.org/W1989220366","https://openalex.org/W2083604559","https://openalex.org/W2112438052","https://openalex.org/W2126281141","https://openalex.org/W2339562433","https://openalex.org/W2396476612","https://openalex.org/W6689742982","https://openalex.org/W6703967046"],"related_works":["https://openalex.org/W107789802","https://openalex.org/W2601058089","https://openalex.org/W272194934","https://openalex.org/W1482839591","https://openalex.org/W2034977219","https://openalex.org/W2062709414","https://openalex.org/W2038260650","https://openalex.org/W3036922594","https://openalex.org/W2036281895","https://openalex.org/W2281299355"],"abstract_inverted_index":{"\"Evaluation":[0],"as":[1,165,167],"a":[2,7,42,110],"service\"":[3],"(EaaS)":[4],"refers":[5],"to":[6,56,82,130,172],"family":[8],"of":[9,20,76,94,100,140,151],"related":[10],"evaluation":[11,39,48,64],"methodologies":[12],"that":[13,25,38,67,78,136],"enables":[14],"community-wide":[15],"evaluations":[16],"and":[17,72,126],"the":[18,31,34,47,57,68,74,85,92,98,116,120,124,131,137,149,156,159,173],"construction":[19],"test":[21],"collections":[22],"on":[23,84],"documents":[24],"cannot":[26],"be":[27,51,80],"easily":[28],"distributed.":[29],"In":[30,87],"API-based":[32],"approach,":[33],"basic":[35],"idea":[36],"is":[37,66,162],"organizers":[40],"provide":[41],"service":[43],"API":[44,69,95,125,157],"through":[45,155],"which":[46],"task":[49,122],"can":[50,79],"completed,":[52],"without":[53],"providing":[54],"access":[55,129,171],"raw":[58,103,132,174],"collection.":[59,133,175],"One":[60],"concern":[61],"with":[62,123,128],"this":[63,88],"approach":[65],"introduces":[70],"biases":[71],"limits":[73],"diversity":[75],"techniques":[77],"brought":[81],"bear":[83],"problem.":[86],"paper,":[89],"we":[90,114],"tackle":[91],"question":[93],"bias":[96,139],"using":[97],"concept":[99],"retrievability.":[101],"The":[102],"data":[104],"for":[105],"our":[106],"analyses":[107],"come":[108],"from":[109],"naturally-occurring":[111],"experiment":[112],"where":[113],"observed":[115],"same":[117,121],"groups":[118,161],"completing":[119],"also":[127],"We":[134],"find":[135],"retrievability":[138],"runs":[141],"generated":[142],"in":[143],"both":[144],"cases":[145],"are":[146],"comparable.":[147],"Moreover,":[148],"fraction":[150],"relevant":[152],"tweets":[153],"retrieved":[154],"by":[158],"participating":[160],"at":[163],"least":[164],"high":[166],"when":[168],"they":[169],"had":[170]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2020,"cited_by_count":1},{"year":2017,"cited_by_count":3}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}