From f4e83169167b8c2d8c3cce86e716d5659d4aa74a Mon Sep 17 00:00:00 2001 From: KatrionaGoldmann Date: Mon, 27 Jan 2025 15:25:04 +0000 Subject: [PATCH 1/6] Add UKCEH models --- trapdata/api/api.py | 10 +++++++ trapdata/api/models/classification.py | 28 ++++++++++++++++++- trapdata/ml/models/base.py | 2 +- trapdata/ml/models/classification.py | 39 +++++++++++++++++++++++++++ 4 files changed, 77 insertions(+), 2 deletions(-) diff --git a/trapdata/api/api.py b/trapdata/api/api.py index 7c0f8100..1abb27a9 100644 --- a/trapdata/api/api.py +++ b/trapdata/api/api.py @@ -20,6 +20,11 @@ MothClassifierQuebecVermont, MothClassifierTuringAnguilla, MothClassifierTuringCostaRica, + MothClassifierTuringJapan, + MothClassifierTuringKenyaUganda, + MothClassifierTuringMadagascar, + MothClassifierTuringSingapore, + MothClassifierTuringThailand, MothClassifierUKDenmark, ) from .models.localization import APIMothDetector @@ -51,6 +56,11 @@ class SourceImageResponse(pydantic.BaseModel): "uk_denmark_moths_2023": MothClassifierUKDenmark, "costa_rica_moths_turing_2024": MothClassifierTuringCostaRica, "anguilla_moths_turing_2024": MothClassifierTuringAnguilla, + "singapore_moths_turing_2024": MothClassifierTuringSingapore, + "thailand_moths_turing_2024": MothClassifierTuringThailand, + "madagascar_moths_turing_2024": MothClassifierTuringMadagascar, + "kenya-uganda_moths_turing_2024": MothClassifierTuringKenyaUganda, + "japan_moths_turing_2024": MothClassifierTuringJapan, "global_moths_2024": MothClassifierGlobal, } _pipeline_choices = dict(zip(PIPELINE_CHOICES.keys(), list(PIPELINE_CHOICES.keys()))) diff --git a/trapdata/api/models/classification.py b/trapdata/api/models/classification.py index 89a9f1db..53311925 100644 --- a/trapdata/api/models/classification.py +++ b/trapdata/api/models/classification.py @@ -5,7 +5,7 @@ import torch from trapdata.common.logs import logger -from trapdata.ml.models.classification import ( +from trapdata.ml.models.classification import ( # TuringJapanSpeciesClassifier,; TuringKenyaUgandaSpeciesClassifier,; TuringMadagascarSpeciesClassifier,; TuringSingaporeSpeciesClassifier,; TuringThailandSpeciesClassifier, GlobalMothSpeciesClassifier, InferenceBaseClass, MothNonMothClassifier, @@ -188,5 +188,31 @@ class MothClassifierTuringAnguilla(APIMothClassifier, TuringAnguillaSpeciesClass pass +# class MothClassifierTuringJapan(APIMothClassifier, TuringJapanSpeciesClassifier): +# pass + + +# class MothClassifierTuringKenyaUganda( +# APIMothClassifier, TuringKenyaUgandaSpeciesClassifier +# ): +# pass + + +# class MothClassifierTuringMadagascar( +# APIMothClassifier, TuringMadagascarSpeciesClassifier +# ): +# pass + + +# class MothClassifierTuringThailand(APIMothClassifier, TuringThailandSpeciesClassifier): +# pass + + +# class MothClassifierTuringSingapore( +# APIMothClassifier, TuringSingaporeSpeciesClassifier +# ): +# pass + + class MothClassifierGlobal(APIMothClassifier, GlobalMothSpeciesClassifier): pass diff --git a/trapdata/ml/models/base.py b/trapdata/ml/models/base.py index c43ce6b8..7818f8f9 100644 --- a/trapdata/ml/models/base.py +++ b/trapdata/ml/models/base.py @@ -288,7 +288,7 @@ def post_process_batch(self, batch_output): # yield self.post_process_single(item) def save_results( - self, item_ids, batch_output, seconds_per_item: float | None = None + self, item_ids, batch_output, seconds_per_item #: float | None = None ): logger.warn("No save method configured for model. Doing nothing with results") return None diff --git a/trapdata/ml/models/classification.py b/trapdata/ml/models/classification.py index 8735885d..231b2cef 100644 --- a/trapdata/ml/models/classification.py +++ b/trapdata/ml/models/classification.py @@ -353,6 +353,45 @@ class TuringAnguillaSpeciesClassifier(SpeciesClassifier, Resnet50Classifier_Turi labels_path = "https://object-arbutus.cloud.computecanada.ca/ami-models/moths/classification/01_anguilla_data_category_map.json" +class TuringThailandSpeciesClassifier(SpeciesClassifier, Resnet50Classifier_Turing): + name = "Turing Thailand Species Classifier" + description = "Trained on 11th November 2024 by Turing team using Resnet50 model." + weights_path = "~/Downloads/turing-thailand_v01_resnet50_2024-11-21-16-28_state.pt" + labels_path = "~/Downloads/01_thailand_data_category_map.json" + + +class TuringMadagascarSpeciesClassifier(SpeciesClassifier, Resnet50Classifier_Turing): + name = "Turing Madagascar Species Classifier" + description = "Trained on 11th November 2024 by Turing team using Resnet50 model." + weights_path = ( + "~/Downloads/turing-madagascar_v01_resnet50_2024-07-01-13-01_state.pt" + ) + labels_path = "~/Downloads/01_madagascar_data_category_map.json" + + +class TuringKenyaUgandaSpeciesClassifier(SpeciesClassifier, Resnet50Classifier_Turing): + name = "Turing Kenya and Uganda Species Classifier" + description = "Trained on 19th November 2024 by Turing team using Resnet50 model." + weights_path = ( + "~/Downloads/turing-kenya-uganda_v01_resnet50_2024-11-19-18-44_state.pt" + ) + labels_path = "~/Downloads/01_kenya-uganda_data_category_map.json" + + +class TuringJapanSpeciesClassifier(SpeciesClassifier, Resnet50Classifier_Turing): + name = "Turing Japan Species Classifier" + description = "Trained on 19th November 2024 by Turing team using Resnet50 model." + weights_path = "~/Downloads/turing-japan_v01_resnet50_2024-11-22-17-22_state.pt" + labels_path = "~/Downloads/01_kenya-uganda_data_category_map.json" + + +class TuringSingaporeSpeciesClassifier(SpeciesClassifier, Resnet50Classifier_Turing): + name = "Turing Singapore Species Classifier" + description = "Trained on 21st November 2024 by Turing team using Resnet50 model." + weights_path = "~/Downloads/turing-singapore_v02_resnet50_2024-11-21-19-58_state.pt" + labels_path = "~/Downloads/02_singapore_data_category_map.json" + + class TuringUKSpeciesClassifier(SpeciesClassifier, Resnet50Classifier_Turing): name = "Turing UK Species Classifier" description = "Trained on 13th May 2024 by Turing team using Resnet50 model." From 564512902358345c2d56459081bb83f35d913786 Mon Sep 17 00:00:00 2001 From: KatrionaGoldmann Date: Mon, 27 Jan 2025 15:26:04 +0000 Subject: [PATCH 2/6] Add UKCEH models --- trapdata/ml/models/classification.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/trapdata/ml/models/classification.py b/trapdata/ml/models/classification.py index 231b2cef..90ce0a90 100644 --- a/trapdata/ml/models/classification.py +++ b/trapdata/ml/models/classification.py @@ -356,40 +356,36 @@ class TuringAnguillaSpeciesClassifier(SpeciesClassifier, Resnet50Classifier_Turi class TuringThailandSpeciesClassifier(SpeciesClassifier, Resnet50Classifier_Turing): name = "Turing Thailand Species Classifier" description = "Trained on 11th November 2024 by Turing team using Resnet50 model." - weights_path = "~/Downloads/turing-thailand_v01_resnet50_2024-11-21-16-28_state.pt" - labels_path = "~/Downloads/01_thailand_data_category_map.json" + weights_path = "https://object-arbutus.cloud.computecanada.ca/ami-models/moths/classification//turing-thailand_v01_resnet50_2024-11-21-16-28_state.pt" + labels_path = "https://object-arbutus.cloud.computecanada.ca/ami-models/moths/classification//01_thailand_data_category_map.json" class TuringMadagascarSpeciesClassifier(SpeciesClassifier, Resnet50Classifier_Turing): name = "Turing Madagascar Species Classifier" description = "Trained on 11th November 2024 by Turing team using Resnet50 model." - weights_path = ( - "~/Downloads/turing-madagascar_v01_resnet50_2024-07-01-13-01_state.pt" - ) - labels_path = "~/Downloads/01_madagascar_data_category_map.json" + weights_path = "https://object-arbutus.cloud.computecanada.ca/ami-models/moths/classification//turing-madagascar_v01_resnet50_2024-07-01-13-01_state.pt" + labels_path = "https://object-arbutus.cloud.computecanada.ca/ami-models/moths/classification//01_madagascar_data_category_map.json" class TuringKenyaUgandaSpeciesClassifier(SpeciesClassifier, Resnet50Classifier_Turing): name = "Turing Kenya and Uganda Species Classifier" description = "Trained on 19th November 2024 by Turing team using Resnet50 model." - weights_path = ( - "~/Downloads/turing-kenya-uganda_v01_resnet50_2024-11-19-18-44_state.pt" - ) - labels_path = "~/Downloads/01_kenya-uganda_data_category_map.json" + weights_path = "https://object-arbutus.cloud.computecanada.ca/ami-models/moths/classification//turing-kenya-uganda_v01_resnet50_2024-11-19-18-44_state.pt" + labels_path = "https://object-arbutus.cloud.computecanada.ca/ami-models/moths/classification//01_kenya-uganda_data_category_map.json" class TuringJapanSpeciesClassifier(SpeciesClassifier, Resnet50Classifier_Turing): name = "Turing Japan Species Classifier" description = "Trained on 19th November 2024 by Turing team using Resnet50 model." - weights_path = "~/Downloads/turing-japan_v01_resnet50_2024-11-22-17-22_state.pt" - labels_path = "~/Downloads/01_kenya-uganda_data_category_map.json" + weights_path = "https://object-arbutus.cloud.computecanada.ca/ami-models/moths/classification//turing-japan_v01_resnet50_2024-11-22-17-22_state.pt" + labels_path = "https://object-arbutus.cloud.computecanada.ca/ami-models/moths/classification//01_kenya-uganda_data_category_map.json" class TuringSingaporeSpeciesClassifier(SpeciesClassifier, Resnet50Classifier_Turing): name = "Turing Singapore Species Classifier" description = "Trained on 21st November 2024 by Turing team using Resnet50 model." - weights_path = "~/Downloads/turing-singapore_v02_resnet50_2024-11-21-19-58_state.pt" - labels_path = "~/Downloads/02_singapore_data_category_map.json" + weights_path = "https://object-arbutus.cloud.computecanada.ca/ami-models/moths/classification//turing-singapore_v02_resnet50_2024-11-21-19-58_state.pt" + labels_path = "https://object-arbutus.cloud.computecanada.ca/ami-models/moths/classification//02_singapore_data_category_map.json" class TuringUKSpeciesClassifier(SpeciesClassifier, Resnet50Classifier_Turing): From 786e6ce9c863b875862d4729afb3ccb27adacaa7 Mon Sep 17 00:00:00 2001 From: KatrionaGoldmann Date: Mon, 27 Jan 2025 16:05:35 +0000 Subject: [PATCH 3/6] uncomment models --- trapdata/api/models/classification.py | 39 +++++++++++++++------------ 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/trapdata/api/models/classification.py b/trapdata/api/models/classification.py index 53311925..eaef1ded 100644 --- a/trapdata/api/models/classification.py +++ b/trapdata/api/models/classification.py @@ -5,7 +5,7 @@ import torch from trapdata.common.logs import logger -from trapdata.ml.models.classification import ( # TuringJapanSpeciesClassifier,; TuringKenyaUgandaSpeciesClassifier,; TuringMadagascarSpeciesClassifier,; TuringSingaporeSpeciesClassifier,; TuringThailandSpeciesClassifier, +from trapdata.ml.models.classification import ( GlobalMothSpeciesClassifier, InferenceBaseClass, MothNonMothClassifier, @@ -14,6 +14,11 @@ QuebecVermontMothSpeciesClassifier2024, TuringAnguillaSpeciesClassifier, TuringCostaRicaSpeciesClassifier, + TuringJapanSpeciesClassifier, + TuringKenyaUgandaSpeciesClassifier, + TuringMadagascarSpeciesClassifier, + TuringSingaporeSpeciesClassifier, + TuringThailandSpeciesClassifier, UKDenmarkMothSpeciesClassifier2024, ) @@ -188,30 +193,30 @@ class MothClassifierTuringAnguilla(APIMothClassifier, TuringAnguillaSpeciesClass pass -# class MothClassifierTuringJapan(APIMothClassifier, TuringJapanSpeciesClassifier): -# pass +class MothClassifierTuringJapan(APIMothClassifier, TuringJapanSpeciesClassifier): + pass -# class MothClassifierTuringKenyaUganda( -# APIMothClassifier, TuringKenyaUgandaSpeciesClassifier -# ): -# pass +class MothClassifierTuringKenyaUganda( + APIMothClassifier, TuringKenyaUgandaSpeciesClassifier +): + pass -# class MothClassifierTuringMadagascar( -# APIMothClassifier, TuringMadagascarSpeciesClassifier -# ): -# pass +class MothClassifierTuringMadagascar( + APIMothClassifier, TuringMadagascarSpeciesClassifier +): + pass -# class MothClassifierTuringThailand(APIMothClassifier, TuringThailandSpeciesClassifier): -# pass +class MothClassifierTuringThailand(APIMothClassifier, TuringThailandSpeciesClassifier): + pass -# class MothClassifierTuringSingapore( -# APIMothClassifier, TuringSingaporeSpeciesClassifier -# ): -# pass +class MothClassifierTuringSingapore( + APIMothClassifier, TuringSingaporeSpeciesClassifier +): + pass class MothClassifierGlobal(APIMothClassifier, GlobalMothSpeciesClassifier): From e064385d2cc1e788d83eecca136a480fe679a785 Mon Sep 17 00:00:00 2001 From: KatrionaGoldmann Date: Mon, 27 Jan 2025 16:39:17 +0000 Subject: [PATCH 4/6] fix catch --- trapdata/ml/models/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trapdata/ml/models/base.py b/trapdata/ml/models/base.py index b496c981..a8964d08 100644 --- a/trapdata/ml/models/base.py +++ b/trapdata/ml/models/base.py @@ -288,7 +288,7 @@ def post_process_batch(self, batch_output): # yield self.post_process_single(item) def save_results( - self, item_ids, batch_output, seconds_per_item #: float | None = None + self, item_ids, batch_output, seconds_per_item: float | None = None ): logger.warn("No save method configured for model. Doing nothing with results") return None From 73f1e7e06db78016ea9d4ef8871310a889b1cc72 Mon Sep 17 00:00:00 2001 From: Katriona Goldmann Date: Tue, 28 Jan 2025 11:46:04 +0000 Subject: [PATCH 5/6] wrong labels file --- trapdata/ml/models/classification.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trapdata/ml/models/classification.py b/trapdata/ml/models/classification.py index 378179e0..1a7fd650 100644 --- a/trapdata/ml/models/classification.py +++ b/trapdata/ml/models/classification.py @@ -378,7 +378,7 @@ class TuringJapanSpeciesClassifier(SpeciesClassifier, Resnet50Classifier_Turing) name = "Turing Japan Species Classifier" description = "Trained on 19th November 2024 by Turing team using Resnet50 model." weights_path = "https://object-arbutus.cloud.computecanada.ca/ami-models/moths/classification//turing-japan_v01_resnet50_2024-11-22-17-22_state.pt" - labels_path = "https://object-arbutus.cloud.computecanada.ca/ami-models/moths/classification//01_kenya-uganda_data_category_map.json" + labels_path = "https://object-arbutus.cloud.computecanada.ca/ami-models/moths/classification//01_japan_data_category_map.json" class TuringSingaporeSpeciesClassifier(SpeciesClassifier, Resnet50Classifier_Turing): From 79ba87d3b551c026392df0bd4a8f379e8cbc9f97 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Tue, 15 Apr 2025 00:14:15 -0700 Subject: [PATCH 6/6] Only add models that have weights available (Kenya) --- trapdata/api/api.py | 8 ----- trapdata/api/models/classification.py | 24 -------------- trapdata/ml/models/classification.py | 48 +++++++++------------------ 3 files changed, 16 insertions(+), 64 deletions(-) diff --git a/trapdata/api/api.py b/trapdata/api/api.py index 454b360a..370d9ffc 100644 --- a/trapdata/api/api.py +++ b/trapdata/api/api.py @@ -22,11 +22,7 @@ MothClassifierQuebecVermont, MothClassifierTuringAnguilla, MothClassifierTuringCostaRica, - MothClassifierTuringJapan, MothClassifierTuringKenyaUganda, - MothClassifierTuringMadagascar, - MothClassifierTuringSingapore, - MothClassifierTuringThailand, MothClassifierUKDenmark, ) from .models.localization import APIMothDetector @@ -51,11 +47,7 @@ "uk_denmark_moths_2023": MothClassifierUKDenmark, "costa_rica_moths_turing_2024": MothClassifierTuringCostaRica, "anguilla_moths_turing_2024": MothClassifierTuringAnguilla, - "singapore_moths_turing_2024": MothClassifierTuringSingapore, - "thailand_moths_turing_2024": MothClassifierTuringThailand, - "madagascar_moths_turing_2024": MothClassifierTuringMadagascar, "kenya-uganda_moths_turing_2024": MothClassifierTuringKenyaUganda, - "japan_moths_turing_2024": MothClassifierTuringJapan, "global_moths_2024": MothClassifierGlobal, "moth_binary": MothClassifierBinary, "insect_orders_2025": InsectOrderClassifier, diff --git a/trapdata/api/models/classification.py b/trapdata/api/models/classification.py index 40a62ce9..2bbc808c 100644 --- a/trapdata/api/models/classification.py +++ b/trapdata/api/models/classification.py @@ -15,11 +15,7 @@ QuebecVermontMothSpeciesClassifier2024, TuringAnguillaSpeciesClassifier, TuringCostaRicaSpeciesClassifier, - TuringJapanSpeciesClassifier, TuringKenyaUgandaSpeciesClassifier, - TuringMadagascarSpeciesClassifier, - TuringSingaporeSpeciesClassifier, - TuringThailandSpeciesClassifier, UKDenmarkMothSpeciesClassifier2024, ) @@ -192,32 +188,12 @@ class MothClassifierTuringAnguilla(APIMothClassifier, TuringAnguillaSpeciesClass pass -class MothClassifierTuringJapan(APIMothClassifier, TuringJapanSpeciesClassifier): - pass - - class MothClassifierTuringKenyaUganda( APIMothClassifier, TuringKenyaUgandaSpeciesClassifier ): pass -class MothClassifierTuringMadagascar( - APIMothClassifier, TuringMadagascarSpeciesClassifier -): - pass - - -class MothClassifierTuringThailand(APIMothClassifier, TuringThailandSpeciesClassifier): - pass - - -class MothClassifierTuringSingapore( - APIMothClassifier, TuringSingaporeSpeciesClassifier -): - pass - - class MothClassifierGlobal(APIMothClassifier, GlobalMothSpeciesClassifier): pass diff --git a/trapdata/ml/models/classification.py b/trapdata/ml/models/classification.py index 082aa09d..b7bf833b 100644 --- a/trapdata/ml/models/classification.py +++ b/trapdata/ml/models/classification.py @@ -406,43 +406,27 @@ class TuringCostaRicaSpeciesClassifier(SpeciesClassifier, Resnet50Classifier_Tur class TuringAnguillaSpeciesClassifier(SpeciesClassifier, Resnet50Classifier_Turing): name = "Turing Anguilla Species Classifier" description = "Trained on 28th June 2024 by Turing team using Resnet50 model." - weights_path = "https://object-arbutus.cloud.computecanada.ca/ami-models/moths/classification/turing-anguilla_v01_resnet50_2024-06-28-17-01_state.pt" - labels_path = "https://object-arbutus.cloud.computecanada.ca/ami-models/moths/classification/01_anguilla_data_category_map.json" - - -class TuringThailandSpeciesClassifier(SpeciesClassifier, Resnet50Classifier_Turing): - name = "Turing Thailand Species Classifier" - description = "Trained on 11th November 2024 by Turing team using Resnet50 model." - weights_path = "https://object-arbutus.cloud.computecanada.ca/ami-models/moths/classification//turing-thailand_v01_resnet50_2024-11-21-16-28_state.pt" - labels_path = "https://object-arbutus.cloud.computecanada.ca/ami-models/moths/classification//01_thailand_data_category_map.json" - - -class TuringMadagascarSpeciesClassifier(SpeciesClassifier, Resnet50Classifier_Turing): - name = "Turing Madagascar Species Classifier" - description = "Trained on 11th November 2024 by Turing team using Resnet50 model." - weights_path = "https://object-arbutus.cloud.computecanada.ca/ami-models/moths/classification//turing-madagascar_v01_resnet50_2024-07-01-13-01_state.pt" - labels_path = "https://object-arbutus.cloud.computecanada.ca/ami-models/moths/classification//01_madagascar_data_category_map.json" + weights_path = ( + "https://object-arbutus.cloud.computecanada.ca/ami-models/moths/classification/" + "turing-anguilla_v01_resnet50_2024-06-28-17-01_state.pt" + ) + labels_path = ( + "https://object-arbutus.cloud.computecanada.ca/ami-models/moths/classification/" + "01_anguilla_data_category_map.json" + ) class TuringKenyaUgandaSpeciesClassifier(SpeciesClassifier, Resnet50Classifier_Turing): name = "Turing Kenya and Uganda Species Classifier" description = "Trained on 19th November 2024 by Turing team using Resnet50 model." - weights_path = "https://object-arbutus.cloud.computecanada.ca/ami-models/moths/classification//turing-kenya-uganda_v01_resnet50_2024-11-19-18-44_state.pt" - labels_path = "https://object-arbutus.cloud.computecanada.ca/ami-models/moths/classification//01_kenya-uganda_data_category_map.json" - - -class TuringJapanSpeciesClassifier(SpeciesClassifier, Resnet50Classifier_Turing): - name = "Turing Japan Species Classifier" - description = "Trained on 19th November 2024 by Turing team using Resnet50 model." - weights_path = "https://object-arbutus.cloud.computecanada.ca/ami-models/moths/classification//turing-japan_v01_resnet50_2024-11-22-17-22_state.pt" - labels_path = "https://object-arbutus.cloud.computecanada.ca/ami-models/moths/classification//01_japan_data_category_map.json" - - -class TuringSingaporeSpeciesClassifier(SpeciesClassifier, Resnet50Classifier_Turing): - name = "Turing Singapore Species Classifier" - description = "Trained on 21st November 2024 by Turing team using Resnet50 model." - weights_path = "https://object-arbutus.cloud.computecanada.ca/ami-models/moths/classification//turing-singapore_v02_resnet50_2024-11-21-19-58_state.pt" - labels_path = "https://object-arbutus.cloud.computecanada.ca/ami-models/moths/classification//02_singapore_data_category_map.json" + weights_path = ( + "https://object-arbutus.cloud.computecanada.ca/ami-models/moths/classification/" + "turing-kenya-uganda_v01_resnet50_2024-11-19-18-44_state.pt" + ) + labels_path = ( + "https://object-arbutus.cloud.computecanada.ca/ami-models/moths/classification/" + "01_kenya-uganda_data_category_map.json" + ) class TuringUKSpeciesClassifier(SpeciesClassifier, Resnet50Classifier_Turing):