{"id":"https://openalex.org/W3153600274","doi":"https://doi.org/10.1145/3442381.3449856","title":"Revisiting the Evaluation Protocol of Knowledge Graph Completion Methods for Link Prediction","display_name":"Revisiting the Evaluation Protocol of Knowledge Graph Completion Methods for Link Prediction","publication_year":2021,"publication_date":"2021-04-19","ids":{"openalex":"https://openalex.org/W3153600274","doi":"https://doi.org/10.1145/3442381.3449856","mag":"3153600274"},"language":"en","primary_location":{"id":"doi:10.1145/3442381.3449856","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3442381.3449856","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Web Conference 2021","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3442381.3449856","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5052462869","display_name":"Sudhanshu Prakash Tiwari","orcid":null},"institutions":[{"id":"https://openalex.org/I155173764","display_name":"Rochester Institute of Technology","ror":"https://ror.org/00v4yb702","country_code":"US","type":"education","lineage":["https://openalex.org/I155173764"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Sudhanshu Tiwari","raw_affiliation_strings":["Rochester Institute of Technology, USA"],"affiliations":[{"raw_affiliation_string":"Rochester Institute of Technology, USA","institution_ids":["https://openalex.org/I155173764"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010266524","display_name":"Iti Bansal","orcid":null},"institutions":[{"id":"https://openalex.org/I155173764","display_name":"Rochester Institute of Technology","ror":"https://ror.org/00v4yb702","country_code":"US","type":"education","lineage":["https://openalex.org/I155173764"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Iti Bansal","raw_affiliation_strings":["Rochester Institute of Technology, USA"],"affiliations":[{"raw_affiliation_string":"Rochester Institute of Technology, USA","institution_ids":["https://openalex.org/I155173764"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5032959798","display_name":"Carlos R. Rivero","orcid":"https://orcid.org/0000-0002-5770-950X"},"institutions":[{"id":"https://openalex.org/I155173764","display_name":"Rochester Institute of Technology","ror":"https://ror.org/00v4yb702","country_code":"US","type":"education","lineage":["https://openalex.org/I155173764"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Carlos R. Rivero","raw_affiliation_strings":["Rochester Institute of Technology, USA"],"affiliations":[{"raw_affiliation_string":"Rochester Institute of Technology, USA","institution_ids":["https://openalex.org/I155173764"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5052462869"],"corresponding_institution_ids":["https://openalex.org/I155173764"],"apc_list":null,"apc_paid":null,"fwci":1.0878,"has_fulltext":false,"cited_by_count":9,"citation_normalized_percentile":{"value":0.81286562,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"809","last_page":"820"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10064","display_name":"Complex Network Analysis Techniques","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/3109","display_name":"Statistical and Nonlinear Physics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.994700014591217,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6562356352806091},{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.6536400318145752},{"id":"https://openalex.org/keywords/bounded-function","display_name":"Bounded function","score":0.5200855135917664},{"id":"https://openalex.org/keywords/predicate","display_name":"Predicate (mathematical logic)","score":0.5005559921264648},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.4700029492378235},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.46354764699935913},{"id":"https://openalex.org/keywords/graph","display_name":"Graph","score":0.44648364186286926},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.4218713641166687},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3975066542625427},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.38762763142585754},{"id":"https://openalex.org/keywords/theoretical-computer-science","display_name":"Theoretical computer science","score":0.3524605631828308},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.24056687951087952}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6562356352806091},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.6536400318145752},{"id":"https://openalex.org/C34388435","wikidata":"https://www.wikidata.org/wiki/Q2267362","display_name":"Bounded function","level":2,"score":0.5200855135917664},{"id":"https://openalex.org/C140146324","wikidata":"https://www.wikidata.org/wiki/Q1144319","display_name":"Predicate (mathematical logic)","level":2,"score":0.5005559921264648},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.4700029492378235},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.46354764699935913},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.44648364186286926},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.4218713641166687},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3975066542625427},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.38762763142585754},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3524605631828308},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.24056687951087952},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3442381.3449856","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3442381.3449856","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Web Conference 2021","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3442381.3449856","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3442381.3449856","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Web Conference 2021","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.4399999976158142,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":43,"referenced_works":["https://openalex.org/W277886906","https://openalex.org/W1533230146","https://openalex.org/W1578543696","https://openalex.org/W1974475152","https://openalex.org/W1993784358","https://openalex.org/W2016753842","https://openalex.org/W2081580037","https://openalex.org/W2094728533","https://openalex.org/W2107306718","https://openalex.org/W2127426251","https://openalex.org/W2146008005","https://openalex.org/W2250184916","https://openalex.org/W2283196293","https://openalex.org/W2300469216","https://openalex.org/W2612900438","https://openalex.org/W2728059831","https://openalex.org/W2785761199","https://openalex.org/W2794432940","https://openalex.org/W2889583850","https://openalex.org/W2891112820","https://openalex.org/W2909137510","https://openalex.org/W2952286769","https://openalex.org/W2964116313","https://openalex.org/W2964221236","https://openalex.org/W2983867426","https://openalex.org/W2984815721","https://openalex.org/W2994634959","https://openalex.org/W2995925258","https://openalex.org/W3003897916","https://openalex.org/W3006415306","https://openalex.org/W3013778764","https://openalex.org/W3014490432","https://openalex.org/W3032390337","https://openalex.org/W3034374701","https://openalex.org/W3037161253","https://openalex.org/W3045272257","https://openalex.org/W3046075728","https://openalex.org/W3090794209","https://openalex.org/W3094451405","https://openalex.org/W3099836348","https://openalex.org/W3176996944","https://openalex.org/W4297684331","https://openalex.org/W6614559876"],"related_works":["https://openalex.org/W4238897586","https://openalex.org/W435179959","https://openalex.org/W2619091065","https://openalex.org/W2059640416","https://openalex.org/W1490753184","https://openalex.org/W2803139007","https://openalex.org/W2088647418","https://openalex.org/W51364034","https://openalex.org/W2898073868","https://openalex.org/W4284663758"],"abstract_inverted_index":{"Completion":[0],"methods":[1,346,365],"learn":[2],"models":[3],"to":[4,45,79,110,248,294],"infer":[5],"missing":[6],"(subject,":[7],"predicate,":[8],"object)":[9],"triples":[10,29,64],"in":[11,114,184,239,285],"knowledge":[12],"graphs,":[13],"a":[14,55,122,166,209,218,259,279,291,306,316,332],"task":[15],"known":[16],"as":[17,254],"link":[18,115,146,252,367],"prediction.":[19],"The":[20,34,58],"training":[21,186,296],"phase":[22,36],"is":[23,98,197,369],"based":[24,49,164,200,304,330],"on":[25,50,165,201,305,331],"samples":[26],"of":[27,38,125,132,151,251,261,269,278,360,363],"positive":[28,41,63,118,134],"and":[30,73,182,281,349],"their":[31],"negative":[32,47,126,270],"counterparts.":[33,271],"test":[35,241],"consists":[37],"ranking":[39,133],"each":[40,96,117],"triple":[42,119],"with":[43],"respect":[44],"its":[46,202,321],"counterparts":[48],"the":[51,102,111,130,152,179,185,190,205,228,240,249,262,267,275,286,300,361],"scores":[52],"obtained":[53],"by":[54,101,231],"learned":[56,317],"model.":[57],"best":[59],"model":[60,196,318],"ranks":[61,216,229,323,329],"all":[62,237],"first.":[65],"Metrics":[66],"like":[67],"mean":[68,70,263],"rank,":[69],"reciprocal":[71],"rank":[72,264],"hits":[74],"at":[75],"k":[76],"are":[77,108,161,324],"used":[78],"assess":[80],"accuracy.":[81],"Under":[82],"this":[83,244],"generic":[84,153],"evaluation":[85,154,250],"protocol,":[86],"we":[87,246,314],"observe":[88],"several":[89,232],"shortcomings:":[90],"1)":[91,256],"Current":[92],"metrics":[93,214],"assume":[94],"that":[95,143,265,298,344,357],"measurement":[97],"upper":[99],"bounded":[100],"same":[103],"constant":[104],"value":[105],"and,":[106],"therefore,":[107],"oblivious":[109],"fact":[112],"that,":[113],"prediction,":[116],"may":[120,222],"have":[121,172],"different":[123,326],"number":[124,268],"counterparts,":[127],"which":[128,160,176,234,355],"alters":[129,178],"difficulty":[131],"triples.":[135],"2)":[136,272],"Benchmarking":[137,170],"datasets":[138,171,342],"contain":[139],"anomalies":[140],"(unrealistic":[141],"redundancy)":[142],"allegedly":[144],"simplifies":[145],"prediction;":[147],"however,":[148,212],"current":[149],"instantiations":[150],"protocol":[155],"do":[156],"not":[157,188],"integrate":[158,282],"anomalies,":[159],"just":[162],"discarded":[163],"user-defined":[167],"threshold.":[168],"3)":[169,288],"been":[173],"randomly":[174],"split,":[175],"typically":[177,198],"graph":[180,302],"topology":[181,303],"results":[183],"split":[187,207],"resembling":[189],"original":[191,301],"dataset.":[192],"4)":[193,311],"A":[194],"single":[195,219],"kept":[199],"accuracy":[203,362],"over":[204,339],"validation":[206],"using":[208],"given":[210],"metric;":[211],"since":[213],"aggregate":[215],"into":[217],"value,":[220],"there":[221],"be":[223,236],"no":[224],"significant":[225],"differences":[226],"among":[227],"produced":[230],"models,":[233],"must":[235],"evaluated":[238],"phase.":[242],"In":[243],"paper,":[245],"contribute":[247],"prediction":[253,368],"follows:":[255],"We":[257,273,289],"propose":[258,290],"variation":[260],"considers":[266],"define":[274],"anomaly":[276],"coefficient":[277,284],"predicate":[280],"such":[283],"protocol.":[287],"downscaling":[292],"algorithm":[293],"generate":[295],"splits":[297],"reflect":[299],"nonparametric,":[307,333],"unpaired":[308],"statistical":[309,335],"test.":[310,336],"During":[312],"validation,":[313],"discard":[315],"only":[319],"if":[320],"output":[322],"significantly":[325,351],"than":[327],"other":[328],"paired":[334],"Our":[337],"experiments":[338],"seven":[340],"well-known":[341],"show":[343],"translation-based":[345],"(TransD,":[347],"TransE":[348],"TransH)":[350],"outperform":[352],"recent":[353],"methods,":[354],"entails":[356],"our":[358],"understanding":[359],"completion":[364],"for":[366],"far":[370],"from":[371],"perfect.":[372]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":5},{"year":2021,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}