Skip to content

Commit 8468a04

Browse files
cryptal-mccryptal-mc
authored
Merge pull request #181 from macrocosm-os/dev
Release 4.5.1
2 parents fde1e89 + 521d012 commit 8468a04

File tree

5 files changed

+56
-31
lines changed

5 files changed

+56
-31
lines changed

‎constants/__init__.py

+44-23
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,7 @@
2424
ModelConstraints,
2525
NormValidationConstraints,
2626
)
27-
from taoverse.model.competition.epsilon import (
28-
FixedEpsilon,
29-
LinearDecay
30-
)
27+
from taoverse.model.competition.epsilon import FixedEpsilon, LinearDecay
3128
from competitions.data import CompetitionId
3229

3330
from typing import Dict, List, Tuple
@@ -37,7 +34,7 @@
3734
# ---------------------------------
3835

3936
# Release
40-
__version__ = "4.5.0"
37+
__version__ = "4.5.1"
4138

4239
# Validator schema version
4340
__validator_version__ = "3.2.0"
@@ -98,7 +95,7 @@
9895
DATASET_BY_COMPETITION_ID: Dict[CompetitionId, str] = {
9996
CompetitionId.M772_MODEL: pt.dataset.SubsetFalconLoader,
10097
CompetitionId.B3_MODEL: pt.dataset.SubsetFalconLoader,
101-
CompetitionId.B7_MODEL: pt.dataset.SubsetFineWebEdu2Loader,
98+
CompetitionId.B7_MODEL: pt.dataset.SubsetFineWebEdu2Loader,
10299
CompetitionId.B14_MODEL: pt.dataset.SubsetFineWebEdu2Loader,
103100
}
104101

@@ -159,7 +156,9 @@
159156
}
160157

161158
# Defined model constraints by competition id with decaying epsilon
162-
MODEL_CONSTRAINTS_BY_COMPETITION_ID_LINEAR_DECAY: Dict[CompetitionId, ModelConstraints] = {
159+
MODEL_CONSTRAINTS_BY_COMPETITION_ID_LINEAR_DECAY: Dict[
160+
CompetitionId, ModelConstraints
161+
] = {
163162
CompetitionId.M772_MODEL: ModelConstraints(
164163
max_model_parameter_size=772_000_000,
165164
min_model_parameter_size=572_000_000,
@@ -215,7 +214,9 @@
215214
}
216215

217216
# Defined model constraints by competition id with decaying epsilon
218-
MODEL_CONSTRAINTS_BY_COMPETITION_ID_LINEAR_DECAY_2: Dict[CompetitionId, ModelConstraints] = {
217+
MODEL_CONSTRAINTS_BY_COMPETITION_ID_LINEAR_DECAY_2: Dict[
218+
CompetitionId, ModelConstraints
219+
] = {
219220
CompetitionId.M772_MODEL: ModelConstraints(
220221
max_model_parameter_size=772_000_000,
221222
min_model_parameter_size=572_000_000,
@@ -251,7 +252,7 @@
251252
"attn_implementation": "flash_attention_2",
252253
},
253254
eval_block_delay=0,
254-
epsilon_func=LinearDecay(0.005, 0.0001, 100800),
255+
epsilon_func=LinearDecay(0.005, 0.0001, 50400),
255256
max_bytes=29 * 1024 * 1024 * 1024,
256257
),
257258
}
@@ -309,22 +310,30 @@
309310
[
310311
Competition(
311312
CompetitionId.M772_MODEL,
312-
MODEL_CONSTRAINTS_BY_COMPETITION_ID_LINEAR_DECAY[CompetitionId.M772_MODEL],
313+
MODEL_CONSTRAINTS_BY_COMPETITION_ID_LINEAR_DECAY[
314+
CompetitionId.M772_MODEL
315+
],
313316
0.14,
314317
),
315318
Competition(
316319
CompetitionId.B3_MODEL,
317-
MODEL_CONSTRAINTS_BY_COMPETITION_ID_LINEAR_DECAY[CompetitionId.B3_MODEL],
320+
MODEL_CONSTRAINTS_BY_COMPETITION_ID_LINEAR_DECAY[
321+
CompetitionId.B3_MODEL
322+
],
318323
0.29,
319324
),
320325
Competition(
321326
CompetitionId.B7_MODEL,
322-
MODEL_CONSTRAINTS_BY_COMPETITION_ID_LINEAR_DECAY[CompetitionId.B7_MODEL],
327+
MODEL_CONSTRAINTS_BY_COMPETITION_ID_LINEAR_DECAY[
328+
CompetitionId.B7_MODEL
329+
],
323330
0.15,
324331
),
325332
Competition(
326333
CompetitionId.B14_MODEL,
327-
MODEL_CONSTRAINTS_BY_COMPETITION_ID_LINEAR_DECAY[CompetitionId.B14_MODEL],
334+
MODEL_CONSTRAINTS_BY_COMPETITION_ID_LINEAR_DECAY[
335+
CompetitionId.B14_MODEL
336+
],
328337
0.42,
329338
),
330339
],
@@ -334,17 +343,23 @@
334343
[
335344
Competition(
336345
CompetitionId.M772_MODEL,
337-
MODEL_CONSTRAINTS_BY_COMPETITION_ID_LINEAR_DECAY[CompetitionId.M772_MODEL],
346+
MODEL_CONSTRAINTS_BY_COMPETITION_ID_LINEAR_DECAY[
347+
CompetitionId.M772_MODEL
348+
],
338349
0.14,
339350
),
340351
Competition(
341352
CompetitionId.B3_MODEL,
342-
MODEL_CONSTRAINTS_BY_COMPETITION_ID_LINEAR_DECAY[CompetitionId.B3_MODEL],
353+
MODEL_CONSTRAINTS_BY_COMPETITION_ID_LINEAR_DECAY[
354+
CompetitionId.B3_MODEL
355+
],
343356
0.29,
344357
),
345358
Competition(
346359
CompetitionId.B14_MODEL,
347-
MODEL_CONSTRAINTS_BY_COMPETITION_ID_LINEAR_DECAY[CompetitionId.B14_MODEL],
360+
MODEL_CONSTRAINTS_BY_COMPETITION_ID_LINEAR_DECAY[
361+
CompetitionId.B14_MODEL
362+
],
348363
0.57,
349364
),
350365
],
@@ -354,23 +369,27 @@
354369
[
355370
Competition(
356371
CompetitionId.M772_MODEL,
357-
MODEL_CONSTRAINTS_BY_COMPETITION_ID_LINEAR_DECAY_2[CompetitionId.M772_MODEL],
372+
MODEL_CONSTRAINTS_BY_COMPETITION_ID_LINEAR_DECAY_2[
373+
CompetitionId.M772_MODEL
374+
],
358375
0.14,
359376
),
360377
Competition(
361378
CompetitionId.B3_MODEL,
362-
MODEL_CONSTRAINTS_BY_COMPETITION_ID_LINEAR_DECAY_2[CompetitionId.B3_MODEL],
379+
MODEL_CONSTRAINTS_BY_COMPETITION_ID_LINEAR_DECAY_2[
380+
CompetitionId.B3_MODEL
381+
],
363382
0.29,
364383
),
365384
Competition(
366385
CompetitionId.B14_MODEL,
367-
MODEL_CONSTRAINTS_BY_COMPETITION_ID_LINEAR_DECAY_2[CompetitionId.B14_MODEL],
386+
MODEL_CONSTRAINTS_BY_COMPETITION_ID_LINEAR_DECAY_2[
387+
CompetitionId.B14_MODEL
388+
],
368389
0.57,
369390
),
370391
],
371392
),
372-
373-
374393
]
375394

376395
for block_and_competitions in COMPETITION_SCHEDULE_BY_BLOCK:
@@ -400,15 +419,17 @@
400419

401420
# validators number of pages to eval over miners on each step.
402421
pages_per_eval_unpack = 5 # With sample unpacking
403-
pages_per_eval_pack = 18
422+
pages_per_eval_pack = 11
404423

405424
# validator eval batch size.
406425
batch_size = 1
407426
# validator eval batch min to keep for next loop.
408427
sample_min = 5
409428
# Max number of uids that can be either pending eval or currently being evaluated.
410429
# We allow the sample_min per competition + 10 additional models to be held at any one time.
411-
updated_models_limit = sample_min * len(MODEL_CONSTRAINTS_BY_COMPETITION_ID_LINEAR_DECAY_2) + 10
430+
updated_models_limit = (
431+
sample_min * len(MODEL_CONSTRAINTS_BY_COMPETITION_ID_LINEAR_DECAY_2) + 10
432+
)
412433
# time required between updates to the chain.
413434
chain_update_cadence = dt.timedelta(minutes=20)
414435
# Number of blocks required between retrying evaluation of a model.

‎neurons/miner.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -278,7 +278,7 @@ async def main(config: bt.config):
278278

279279
# Init model.
280280
# Init model.
281-
tokenizer = ft.model.load_tokenizer(model_constraints, cache_dir=config.model_dir)
281+
tokenizer = pt.model.load_tokenizer(model_constraints, cache_dir=config.model_dir)
282282
model = await load_starting_model(config, metagraph, chain_metadata_store, kwargs)
283283
model = model.train()
284284
model = model.to(config.device)
@@ -410,11 +410,11 @@ async def main(config: bt.config):
410410
)
411411

412412
# First, reload the best model from the training run.
413-
model_to_upload = ft.mining.load_local_model(
413+
model_to_upload = pt.mining.load_local_model(
414414
model_dir, model_constraints.kwargs
415415
)
416416

417-
await ft.mining.push(
417+
await pt.mining.push(
418418
model_to_upload,
419419
config.hf_repo_id,
420420
wallet,

‎neurons/validator.py

+3-2
Original file line numberDiff line numberDiff line change
@@ -648,7 +648,7 @@ def clean_models(self):
648648

649649
self.local_store.delete_unreferenced_models(
650650
valid_models_by_hotkey=evaluated_hotkeys_to_model_id,
651-
grace_period_seconds=300,
651+
grace_period_seconds=600,
652652
)
653653
except Exception as e:
654654
bt.logging.error(f"Error in clean loop: {e}")
@@ -891,11 +891,12 @@ async def run_step(self):
891891
tokenizer.eos_token_id,
892892
pack_samples,
893893
),
894-
ttl=400,
894+
ttl=430,
895895
mode="spawn",
896896
)
897897

898898
del model_i
899+
899900
except Exception as e:
900901
bt.logging.error(
901902
f"Error in eval loop: {e}. Setting losses for uid: {uid_i} to infinity."

‎pretrain/mining.py

+5-2
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646

4747
from competitions.data import CompetitionId
4848

49+
4950
def model_path(base_dir: str, run_id: str) -> str:
5051
"""
5152
Constructs a file path for storing the model relating to a training run.
@@ -96,7 +97,8 @@ async def push(
9697

9798
bt.logging.debug("Started uploading model to hugging face...")
9899
model_id = await remote_model_store.upload_model(
99-
Model(id=model_id, pt_model=model), model_constraints)
100+
Model(id=model_id, pt_model=model), model_constraints
101+
)
100102

101103
bt.logging.success("Uploaded model to hugging face.")
102104

@@ -190,6 +192,7 @@ def load_local_model(model_dir: str, kwargs: Dict[str, Any]) -> PreTrainedModel:
190192
**kwargs,
191193
)
192194

195+
193196
async def load_remote_model(
194197
uid: int,
195198
download_dir: str,
@@ -245,7 +248,7 @@ async def load_best_model(
245248
remote_model_store: Optional[RemoteModelStore] = None,
246249
) -> PreTrainedModel:
247250
"""Loads the model from the best performing miner to download_dir"""
248-
best_uid = ft.graph.best_uid(competition_id=competition_id)
251+
best_uid = pt.graph.best_uid(competition_id=competition_id)
249252
if best_uid is None:
250253
raise ValueError(f"No best models found for {competition_id}")
251254

‎requirements.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,4 +11,4 @@ transformers==4.44.1
1111
wandb
1212
datasets
1313
flash-attn
14-
taoverse==1.0.5
14+
taoverse==1.0.6

0 commit comments

Comments
 (0)