{"id":"https://openalex.org/W6892523083","doi":"https://doi.org/10.5281/zenodo.11102888","title":"WildCLIP: Scene and animal attribute retrieval from camera trap data with domain-adapted vision-language models","display_name":"WildCLIP: Scene and animal attribute retrieval from camera trap data with domain-adapted vision-language models","publication_year":2024,"publication_date":"2024-05-06","ids":{"openalex":"https://openalex.org/W6892523083","doi":"https://doi.org/10.5281/zenodo.11102888"},"language":"en","primary_location":{"id":"pmh:oai:infoscience.epfl.ch:311775","is_oa":true,"landing_page_url":"https://infoscience.epfl.ch/handle/20.500.14299/208814","pdf_url":null,"source":{"id":"https://openalex.org/S4306400487","display_name":"Infoscience (Ecole Polytechnique F\u00e9d\u00e9rale de Lausanne)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"dataset"},"type":"dataset","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://infoscience.epfl.ch/handle/20.500.14299/208814","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Gabeff, Valentin","orcid":"https://orcid.org/0000-0003-3352-1825"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gabeff, Valentin","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0003-3352-1825","affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Ru\u00dfwurm, Marc","orcid":"https://orcid.org/0000-0001-6612-5744"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ru\u00dfwurm, Marc","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0001-6612-5744","affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Tuia, Devis","orcid":"https://orcid.org/0000-0003-0374-2459"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tuia, Devis","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0003-0374-2459","affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Mathis, Alexander","orcid":"https://orcid.org/0000-0002-3777-2202"},"institutions":[{"id":"https://openalex.org/I5124864","display_name":"\u00c9cole Polytechnique F\u00e9d\u00e9rale de Lausanne","ror":"https://ror.org/02s376052","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I5124864"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Mathis, Alexander","raw_affiliation_strings":["\u00c9cole Polytechnique F\u00e9d\u00e9rale de Lausanne"],"raw_orcid":"https://orcid.org/0000-0002-3777-2202","affiliations":[{"raw_affiliation_string":"\u00c9cole Polytechnique F\u00e9d\u00e9rale de Lausanne","institution_ids":["https://openalex.org/I5124864"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":null,"topics":[],"keywords":[{"id":"https://openalex.org/keywords/snapshot","display_name":"Snapshot (computer storage)","score":0.843999981880188},{"id":"https://openalex.org/keywords/camera-trap","display_name":"Camera trap","score":0.6050999760627747},{"id":"https://openalex.org/keywords/vocabulary","display_name":"Vocabulary","score":0.4986000061035156},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.45890000462532043},{"id":"https://openalex.org/keywords/license","display_name":"License","score":0.4172999858856201},{"id":"https://openalex.org/keywords/structuring","display_name":"Structuring","score":0.396699994802475},{"id":"https://openalex.org/keywords/image-processing","display_name":"Image processing","score":0.3084999918937683}],"concepts":[{"id":"https://openalex.org/C55282118","wikidata":"https://www.wikidata.org/wiki/Q252683","display_name":"Snapshot (computer storage)","level":2,"score":0.843999981880188},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.79339998960495},{"id":"https://openalex.org/C2779101711","wikidata":"https://www.wikidata.org/wiki/Q1723004","display_name":"Camera trap","level":3,"score":0.6050999760627747},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5752000212669373},{"id":"https://openalex.org/C2777601683","wikidata":"https://www.wikidata.org/wiki/Q6499736","display_name":"Vocabulary","level":2,"score":0.4986000061035156},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.45890000462532043},{"id":"https://openalex.org/C2780560020","wikidata":"https://www.wikidata.org/wiki/Q79719","display_name":"License","level":2,"score":0.4172999858856201},{"id":"https://openalex.org/C2775945657","wikidata":"https://www.wikidata.org/wiki/Q381442","display_name":"Structuring","level":2,"score":0.396699994802475},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.3944999873638153},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3359000086784363},{"id":"https://openalex.org/C9417928","wikidata":"https://www.wikidata.org/wiki/Q1070689","display_name":"Image processing","level":3,"score":0.3084999918937683},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.3046000003814697},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.2786000072956085},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2777999937534332},{"id":"https://openalex.org/C2779506182","wikidata":"https://www.wikidata.org/wiki/Q7580141","display_name":"Spotting","level":2,"score":0.272599995136261},{"id":"https://openalex.org/C2776151529","wikidata":"https://www.wikidata.org/wiki/Q3045304","display_name":"Object detection","level":3,"score":0.27239999175071716},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.26579999923706055},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.25920000672340393},{"id":"https://openalex.org/C160633673","wikidata":"https://www.wikidata.org/wiki/Q355198","display_name":"Pixel","level":2,"score":0.2590000033378601}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:infoscience.epfl.ch:311775","is_oa":true,"landing_page_url":"https://infoscience.epfl.ch/handle/20.500.14299/208814","pdf_url":null,"source":{"id":"https://openalex.org/S4306400487","display_name":"Infoscience (Ecole Polytechnique F\u00e9d\u00e9rale de Lausanne)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"dataset"},{"id":"doi:10.5281/zenodo.11102888","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.11102888","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Dataset"}],"best_oa_location":{"id":"pmh:oai:infoscience.epfl.ch:311775","is_oa":true,"landing_page_url":"https://infoscience.epfl.ch/handle/20.500.14299/208814","pdf_url":null,"source":{"id":"https://openalex.org/S4306400487","display_name":"Infoscience (Ecole Polytechnique F\u00e9d\u00e9rale de Lausanne)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"dataset"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"#############":[0,15],"WildCLIP:":[1],"Scene":[2,447],"and":[3,84,113,190,223,247,257,284,311,321,360,416,448,462,465,468,511,514,517,520,523],"animal":[4,449],"attribute":[5,114,450],"retrieval":[6,451],"from":[7,52,352,371,391,411,452],"camera":[8,142,350,364,453,498],"trap":[9,143,454,499],"data":[10,80,111,117,153,213,241,274,303,455,487],"with":[11,49,88,133,206,214,234,242,248,267,275,296,304,312,369,389,409,456],"domain-adapted":[12,457],"vision-language":[13,63,458],"models":[14,64,101,200,425],"Authors:":[16],"Valentin":[17,461],"Gabeff,":[18],"Marc":[19,464],"Russwurm,":[20],"Devis":[21,467],"Tuia":[22],"&":[23],"Alexander":[24],"Mathis":[25],"Affiliation:":[26],"EPFL":[27],"Date:":[28],"January,":[29],"2024":[30],"Link":[31],"to":[32,45,60,82,98,126,309],"the":[33,53,67,76,106,127,151,162,172,178,188,207,235,249,268,297,313,333,337,363,424,432,483],"article:":[34,492],"https://link.springer.com/article/10.1007/s11263-024-02026-6":[35],"--------------------------------":[36],"WildCLIP":[37,92,199],"is":[38,358],"a":[39,134],"fine-tuned":[40],"CLIP":[41,195,204,232,265,294],"model":[42,196,205,233,266,295],"that":[43],"allows":[44],"retrieve":[46],"camera-trap":[47,71],"events":[48],"natural":[50],"language":[51],"Snapshot":[54,78,131,485],"Serengeti":[55,79,132,486],"dataset.":[56],"This":[57],"project":[58,107],"intends":[59],"demonstrate":[61],"how":[62,97,423],"may":[65],"assist":[66],"annotation":[68],"process":[69],"of":[70,91,119,130,150,174,473,501],"datasets.":[72],"Here":[73],"we":[74],"provide":[75,332],"processed":[77],"used":[81,281,318],"train":[83,338,345],"evaluate":[85],"WildCLIP,":[86],"along":[87],"two":[89],"versions":[90],"(model":[93,197],"weights).":[94],"Details":[95,421],"on":[96,157,212,220,240,254,273,286,302,323,422],"run":[99],"these":[100],"can":[102,154,428],"be":[103,155,429],"found":[104,156,430],"in":[105,431,505],"github":[108],"repository.":[109],"Provided":[110],"(images":[112],"annotations):":[115],"The":[116,344,355],"consists":[118],"380":[120,122],"x":[121],"image":[123],"crops":[124],"corresponding":[125],"MegaDetector":[128,179],"output":[129],"confidence":[135],"threshold":[136],"above":[137],"0.7.":[138],"We":[139,169,330],"considered":[140],"only":[141,382,402],"images":[144,500],"containing":[145,336],"single":[146],"individuals.":[147],"A":[148],"description":[149],"original":[152],"LILA":[158,175,353],"here,":[159],"released":[160],"under":[161],"Community":[163],"Data":[164],"License":[165],"Agreement":[166],"(permissive":[167],"variant).":[168],"warmly":[170],"thank":[171],"authors":[173],"for":[176,186,228,262,282,319],"making":[177],"outputs":[180],"publicly":[181],"available,":[182],"as":[183,185],"well":[184],"structuring":[187],"dataset":[189],"facilitating":[191],"its":[192],"access.":[193],"Adapted":[194],"weights):":[198],"provided:":[201],"[New]":[202,230],"WildCLIP_vitb16_t1.pth:":[203],"ViT-B/16":[208,236,269,298],"visual":[209,237,270,299],"backbone":[210,238,271,300],"trained":[211,239,272,285,301,322,427],"captions":[215,243,276,305,370,390,410],"following":[216,244,277,306],"template":[217,245,278,380,400],"1.":[218,279],"Trained":[219,253],"both":[221,255],"base":[222,256,287,324],"novel":[224,258],"vocabulary":[225,259,288,325],"(see":[226,260],"paper":[227,261],"details).":[229,263],"WildCLIP_vitb16_t1_lwf.pth:":[231],"1,":[246,413],"additional":[250,314],"VR-LwF":[251,315],"loss.":[252,316],"WildCLIP_vitb16_t1_base.pth:":[264],"Model":[280,317],"evaluation":[283,320],"only.":[289,326],"(previously":[290,327],"named":[291,328],"WildCLIP_vitb16_t1.pth)":[292],"WildCLIP_vitb16_t1t7_lwf_base.pth:":[293],"templates":[307,372,392,412],"1":[308,373,381,393,401],"7,":[310],"WildCLIP_vitb16_t1t7_lwf.pth)":[329],"also":[331,361,489],"CSV":[334],"files":[335],"/":[339,341,346],"val":[340],"test":[342,347],"splits.":[343],"splits":[348],"follow":[349],"split":[351,357],"(https://lila.science/datasets/snapshot-serengeti).":[354],"validation":[356],"custom,":[359],"at":[362],"level.":[365],"train_dataset_crops_single_animal_template_captions_T1T7_ID.csv:":[366],"Train":[367],"set":[368,388,408],"through":[374,394],"7":[375,395],"(column":[376,383,396,403],"\"all":[377,397,419],"captions\")":[378,398,420],"or":[379,399,441],"\"template":[384,404],"1\")":[385,405],"val_dataset_crops_single_animal_template_captions_T1T7_ID.csv:":[386],"Validation":[387],"test_dataset_crops_single_animal_template_captions_T1T8T10.csv:":[406],"Test":[407],"8,":[414],"9":[415],"10":[417],"(columns":[418],"were":[426],"associated":[433],"publication.":[434],"References:":[435],"If":[436,480],"you":[437,481],"find":[438],"our":[439],"code,":[440],"weights,":[442],"please":[443,488],"cite:":[444],"@article{gabeff2024wildclip,":[445],"title={WildCLIP:":[446],"models},":[459],"author={Gabeff,":[460],"Ru{\\ss}wurm,":[463],"Tuia,":[466],"Mathis,":[469],"Alexander},":[470],"journal={International":[471],"Journal":[472],"Computer":[474],"Vision},":[475],"pages={1--17},":[476],"year={2024},":[477],"publisher={Springer}":[478],"}":[479,535],"use":[482],"adapted":[484],"cite":[490],"their":[491],"@article{swanson2015snapshot,":[493],"title={Snapshot":[494],"Serengeti,":[495],"high-frequency":[496],"annotated":[497],"40":[502],"mammalian":[503],"species":[504],"an":[506],"African":[507],"savanna},":[508],"author={Swanson,":[509],"Alexandra":[510],"Kosmala,":[512],"Margaret":[513],"Lintott,":[515],"Chris":[516],"Simpson,":[518],"Robert":[519],"Smith,":[521],"Arfon":[522],"Packer,":[524],"Craig},":[525],"journal={Scientific":[526],"data},":[527],"volume={2},":[528],"number={1},":[529],"pages={1--14},":[530],"year={2015},":[531],"publisher={Nature":[532],"Publishing":[533],"Group}":[534]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2025-10-10T00:00:00"}