Merge pull request #735 from mlcommons/dev

priyakasimbeg · web-flow · commit 6a3ff16d2b37 · 2024-03-26T17:08:14.000-07:00
Dev -&gt; main
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,23 @@
 # Change Log
 
+## algoperf-benchmark-0.1.4 (2024-03-26)
+
+Upgrade CUDA version to CUDA 12.1:
+- Upgrade CUDA version in Dockerfiles that will be used for scoring.
+- Update Jax and PyTorch package version tags to use local CUDA installation.
+
+Add flag for completely disabling checkpointing. 
+- Note that we will run with checkpointing off at scoring time.
+
+Update Deepspeech and Conformer variant target setting configurations. 
+- Note that variant targets are not final. 
+
+Fixed bug in scoring code to take best trial in a study for external-tuning ruleset.
+
+Added instructions for submission. 
+
+Changed default number of workers for PyTorch data loaders to 0. Running with >0 may lead to incorrect eval results see https://github.com/mlcommons/algorithmic-efficiency/issues/732. 
+
 ## algoperf-benchmark-0.1.2 (2024-03-04)
 Workload variant additions and fixes:
 - Add Deepspeech workload variant
diff --git a/reference_algorithms/target_setting_algorithms/librispeech_conformer_attention_temperature/tuning_search_space.json b/reference_algorithms/target_setting_algorithms/librispeech_conformer_attention_temperature/tuning_search_space.json
@@ -1,27 +1,27 @@
 {
     "learning_rate": {
         "feasible_points": [
-            0.001308209823469072
+            0.0007852999990476642
         ]
     },
     "beta1": {
         "feasible_points": [
-            0.9731333693827139
+            0.6994142393023162
         ]
     },
     "beta2": {
         "feasible_points": [
-            0.9981232922116359
+            0.9918636824608852
         ]
     },
     "warmup_steps": {
         "feasible_points": [
-            9999
+            6000
         ]
     },
     "weight_decay": {
         "feasible_points": [
-            0.16375311233774334
+            0.07286322158086678
         ]
     }
 }
diff --git a/reference_algorithms/target_setting_algorithms/librispeech_conformer_gelu/tuning_search_space.json b/reference_algorithms/target_setting_algorithms/librispeech_conformer_gelu/tuning_search_space.json
@@ -1,17 +1,17 @@
 {
     "learning_rate": {
         "feasible_points": [
-            0.001308209823469072
+            0.000590120167916659
         ]
     },
     "beta1": {
         "feasible_points": [
-            0.9731333693827139
+            0.737199286155609
         ]
     },
     "beta2": {
         "feasible_points": [
-            0.9981232922116359
+            0.05919391544031072
         ]
     },
     "warmup_steps": {
@@ -21,7 +21,7 @@
     },
     "weight_decay": {
         "feasible_points": [
-            0.16375311233774334
+            0.14128519778326312
         ]
     }
 }
diff --git a/reference_algorithms/target_setting_algorithms/librispeech_conformer_layernorm/tuning_search_space.json b/reference_algorithms/target_setting_algorithms/librispeech_conformer_layernorm/tuning_search_space.json
@@ -1,27 +1,27 @@
 {
     "learning_rate": {
         "feasible_points": [
-            0.001308209823469072
+            0.0014446807792420305
         ]
     },
     "beta1": {
         "feasible_points": [
-            0.9731333693827139
+            0.7427148812902895
         ]
     },
     "beta2": {
         "feasible_points": [
-            0.9981232922116359
+            0.8993064520764248
         ]
     },
     "warmup_steps": {
         "feasible_points": [
-            9999
+            3000
         ]
     },
     "weight_decay": {
         "feasible_points": [
-            0.16375311233774334
+            0.06875136511682291
         ]
     }
 }
diff --git a/reference_algorithms/target_setting_algorithms/librispeech_deepspeech_no_resnet/tuning_search_space.json b/reference_algorithms/target_setting_algorithms/librispeech_deepspeech_no_resnet/tuning_search_space.json
@@ -1,27 +1,27 @@
 {
     "learning_rate": {
         "feasible_points": [
-            0.0035278622506232458
+            0.0020162740358935045
         ]
     },
     "beta1": {
         "feasible_points": [
-            0.8192305396005781
+            0.9604907112078142
         ]
     },
     "beta2": {
         "feasible_points": [
-            0.495850879212151
+            0.8765457000160508
         ]
     },
     "warmup_steps": {
         "feasible_points": [
-            6000
+            3600
         ]
     },
     "weight_decay": {
         "feasible_points": [
-            0.04339748256184769
+            0.0006149579248633481
         ]
     }
 }
diff --git a/reference_algorithms/target_setting_algorithms/librispeech_deepspeech_norm_and_spec_aug/tuning_search_space.json b/reference_algorithms/target_setting_algorithms/librispeech_deepspeech_norm_and_spec_aug/tuning_search_space.json
@@ -1,27 +1,27 @@
 {
     "learning_rate": {
         "feasible_points": [
-            0.001308209823469072
+            0.0014446807792420305
         ]
     },
     "beta1": {
         "feasible_points": [
-            0.9731333693827139
+            0.7427148812902895
         ]
     },
     "beta2": {
         "feasible_points": [
-            0.9981232922116359
+            0.8993064520764248
         ]
     },
     "warmup_steps": {
         "feasible_points": [
-            6000
+            1800
         ]
     },
     "weight_decay": {
         "feasible_points": [
-            0.16375311233774334
+            0.06875136511682291
         ]
     }
 }
diff --git a/reference_algorithms/target_setting_algorithms/librispeech_deepspeech_tanh/tuning_search_space.json b/reference_algorithms/target_setting_algorithms/librispeech_deepspeech_tanh/tuning_search_space.json
@@ -1,27 +1,27 @@
 {
     "learning_rate": {
         "feasible_points": [
-            0.003632312571224348
+            0.003604759885558324
         ]
     },
     "beta1": {
         "feasible_points": [
-            0.9980088784197237
+            0.9931094324430452
         ]
     },
     "beta2": {
         "feasible_points": [
-            0.9982275351621527
+            0.9976871843749077
         ]
     },
     "warmup_steps": {
         "feasible_points": [
-            6000
+            720
         ]
     },
     "weight_decay": {
         "feasible_points": [
-            0.2479797019098727
+            0.120077307855989
         ]
     }
 }
diff --git a/utils/target_setting_workload_config.json b/utils/target_setting_workload_config.json
@@ -176,7 +176,7 @@
     "librispeech_conformer_attention_temperature": {
         "max_steps": 80000,
         "dataset": "librispeech",
-        "submission_path": "reference_algorithms/target_setting_algorithms/jax_nadamw.py",
+        "submission_path": "reference_algorithms/target_setting_algorithms/jax_adamw.py",
         "tuning_search_space": "reference_algorithms/target_setting_algorithms/librispeech_conformer_attention_temperature/tuning_search_space.json"
     },
     "librispeech_conformer_gelu": {

Original file line number	Diff line number	Diff line change
`@@ -1,27 +1,27 @@`
`1`	`1`	`{`
`2`	`2`	`"learning_rate": {`
`3`	`3`	`"feasible_points": [`
`4`		`- 0.001308209823469072`
	`4`	`+ 0.0007852999990476642`
`5`	`5`	`]`
`6`	`6`	`},`
`7`	`7`	`"beta1": {`
`8`	`8`	`"feasible_points": [`
`9`		`- 0.9731333693827139`
	`9`	`+ 0.6994142393023162`
`10`	`10`	`]`
`11`	`11`	`},`
`12`	`12`	`"beta2": {`
`13`	`13`	`"feasible_points": [`
`14`		`- 0.9981232922116359`
	`14`	`+ 0.9918636824608852`
`15`	`15`	`]`
`16`	`16`	`},`
`17`	`17`	`"warmup_steps": {`
`18`	`18`	`"feasible_points": [`
`19`		`- 9999`
	`19`	`+ 6000`
`20`	`20`	`]`
`21`	`21`	`},`
`22`	`22`	`"weight_decay": {`
`23`	`23`	`"feasible_points": [`
`24`		`- 0.16375311233774334`
	`24`	`+ 0.07286322158086678`
`25`	`25`	`]`
`26`	`26`	`}`
`27`	`27`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,17 +1,17 @@`
`1`	`1`	`{`
`2`	`2`	`"learning_rate": {`
`3`	`3`	`"feasible_points": [`
`4`		`- 0.001308209823469072`
	`4`	`+ 0.000590120167916659`
`5`	`5`	`]`
`6`	`6`	`},`
`7`	`7`	`"beta1": {`
`8`	`8`	`"feasible_points": [`
`9`		`- 0.9731333693827139`
	`9`	`+ 0.737199286155609`
`10`	`10`	`]`
`11`	`11`	`},`
`12`	`12`	`"beta2": {`
`13`	`13`	`"feasible_points": [`
`14`		`- 0.9981232922116359`
	`14`	`+ 0.05919391544031072`
`15`	`15`	`]`
`16`	`16`	`},`
`17`	`17`	`"warmup_steps": {`
`@@ -21,7 +21,7 @@`
`21`	`21`	`},`
`22`	`22`	`"weight_decay": {`
`23`	`23`	`"feasible_points": [`
`24`		`- 0.16375311233774334`
	`24`	`+ 0.14128519778326312`
`25`	`25`	`]`
`26`	`26`	`}`
`27`	`27`	`}`