diff --git a/.gitignore b/.gitignore index 2526b5b..d981254 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,6 @@ *.pyc *.log *.data -*.txt #Data __pycache__/ diff --git a/README.md b/README.md index af63a16..ef9f20e 100644 --- a/README.md +++ b/README.md @@ -6,32 +6,73 @@ Code for the **Co**ntent Ca**ch**ing algorithm in edge caching. ```shell git clone https://github.com/DarriusL/CacheLab.git +``` + +Create an environment using one of these methods: + +### conda + +```shell cd CacheLab conda env create -f cachelab_dev.yml conda activate cachelab_dev -git update-index --assume-unchanged config/lab_cfg.json -git update-index --assume-unchanged config/data_process_cfg.json ``` +### pip + +```shell +conda create -n CacheLab python=3.11 +pip install -r requirements.txt +``` + + + ## Framework file structure ``` -CacheLab ├── .gitignore ├── cache -│ └── logger -│ └── logger.log ├── cachelab_env.yml ├── config -│ ├── caser/. +│ ├── caser +│ │ ├── caser_appliances.json +│ │ ├── caser_ml1m.json +│ │ └── caser_music.json │ ├── CFG_README.md -│ ├── cl4srec/. +│ ├── cl4srec +│ │ ├── cl4srec_appliances.json +│ │ ├── cl4srec_ml1m.json +│ │ └── cl4srec_music.json │ ├── data_process_cfg.json -│ ├── duo4srec/. -│ ├── ec4srec/. -│ ├── egpc/. +│ ├── duo4srec +│ │ ├── duo4srec_appliances.json +│ │ ├── duo4srec_ml1m.json +│ │ └── duo4srec_music.json +│ ├── ec4srec +│ │ ├── ec4srec_appliances.json +│ │ ├── ec4srec_ml1m.json +│ │ └── ec4srec_music.json +│ ├── egpc +│ │ ├── egpc_appliances.json +│ │ ├── egpc_ml1m.json +│ │ └── egpc_music.json +│ ├── fifo +│ │ ├── fifo_appliances.json +│ │ ├── fifo_ml1m.json +│ │ └── fifo_music.json │ ├── lab_cfg.json -│ └── psac/. +│ ├── lfu +│ │ ├── lfu_appliances.json +│ │ ├── lfu_ml1m.json +│ │ └── lfu_music.json +│ ├── lru +│ │ ├── lru_appliances.json +│ │ ├── lru_ml1m.json +│ │ └── lru_music.json +│ └── psac +│ ├── psac_gen_appliances.json +│ ├── psac_gen_ml1m.json +│ └── psac_gen_music.json ├── data │ ├── augmentation.py │ ├── datasets @@ -73,28 +114,33 @@ CacheLab │ │ └── music_devide_25.data │ ├── generator.py │ ├── processor.py -│ ├── saved/. +│ ├── saved./ │ └── __init__.py -| ├── executor.py ├── lib │ ├── callback.py │ ├── glb_var.py -│ ├── graph_util.py │ ├── json_util.py │ └── util.py -| +├── LICENSE ├── model │ ├── attnet.py │ ├── cnnnet.py -│ └── framework -│ ├── caser.py -│ ├── cl4srec.py -│ ├── duo4srec.py -│ ├── ec4srec.py -│ ├── egpc.py -│ └── psac.py +│ ├── framework +│ │ ├── base.py +│ │ ├── caser.py +│ │ ├── cl4srec.py +│ │ ├── duo4srec.py +│ │ ├── ec4srec.py +│ │ ├── egpc.py +│ │ ├── fifo.py +│ │ ├── lfu.py +│ │ ├── lru.py +│ │ └── psac.py +│ ├── loss.py +│ └── __init__.py ├── README.md +├── requirements.txt └── Room ├── officer.py ├── work.py @@ -126,9 +172,9 @@ If you need to use the provided processed data set, download it to path : ./data Coventional: -- [ ] FIFO -- [ ] LRU -- [ ] LFU +- [x] FIFO +- [x] LRU +- [x] LFU CL-based: @@ -181,6 +227,30 @@ You need to configure the data processing configuration file yourself:@./config/ python executor.py --data_process=True ``` +FIFO + +```shell +python executor.py -sc='./config/fifo/fifo_ml1m.json' --mode=test +python executor.py -sc='./config/fifo/fifo_appliances.json' --mode=test +python executor.py -sc='./config/fifo/fifo_music.json' --mode=test +``` + +LRU + +```shell +python executor.py -sc='./config/lru/lru_ml1m.json' --mode=test +python executor.py -sc='./config/lru/lru_appliances.json' --mode=test +python executor.py -sc='./config/lru/lru_music.json' --mode=test +``` + +LFU + +```shell +python executor.py -sc='./config/lfu/lfu_ml1m.json' --mode=test +python executor.py -sc='./config/lfu/lfu_appliances.json' --mode=test +python executor.py -sc='./config/lfu/lfu_music.json' --mode=test +``` + CL4SRec ```shell diff --git a/Room/officer.py b/Room/officer.py index d16b01a..adccbb9 100644 --- a/Room/officer.py +++ b/Room/officer.py @@ -18,7 +18,7 @@ def get_save_path(cfg): return './data/saved/' + cfg['net']['type'].lower() + '/' + cfg['dataset']['type'] + '/' + f'{cfg["net"]["d"]}_{cfg["net"]["n_kernels"]}/model.model'; elif cfg['net']['type'].lower() in ['fifo','lru','lfu']: return './data/saved/' + cfg['net']['type'].lower() + '/' + cfg['dataset']['type'] + '/model.model'; - elif cfg['net']['is_norm_fist']: + elif cfg['net']['is_norm_first']: norm_type = 'pre'; else: norm_type = 'post'; @@ -88,7 +88,11 @@ class AbstractTester(): def __init__(self, test_cfg_dict, model) -> None: util.set_attr(self, test_cfg_dict); self.device = glb_var.get_value('device'); - self.model = model.to(self.device); + if model.type.lower() not in ['fifo', 'lru', 'lfu']: + self.model = model.to(self.device); + else: + self.model = model; + def test(self, test_data): logger.error('Method needs to be called after being implemented'); @@ -644,7 +648,7 @@ def _report_result(self, result): logger.info(str); return str; -class ConventionalTester(): +class ConventionalTester(AbstractTester): '''Tester for conventional algorithm: FIFO,LRU,LFU Parameters: @@ -652,8 +656,7 @@ class ConventionalTester(): ''' def __init__(self, config, model) -> None: - util.set_attr(self, config['test']); - self.model = model; + super().__init__(config['test'], model); self.cfg = config; if self.save: self.save_path, _ = os.path.split(self.model_save_path); @@ -683,8 +686,8 @@ def _caching_and_cal_qoe_trafficload(self, data, cache_size_list): self.model.clear(); for batch_id in range(batch_size): #su:(slide_len, T) - su = data[batch_id, :].unfold(-1, self.slide_T + 1, self.slide_T)[:, :self.slide_T]; - self.model.update(su.reshape(1, -1)); + su = data[batch_id, :].unfold(-1, self.slide_T + 1, self.slide_T + 1)[:, :self.slide_T]; + self.model.update(su.reshape(-1)); QoE = {}; TrafficLoad = {}; for cache_size in cache_size_list: @@ -697,7 +700,7 @@ def _caching_and_cal_qoe_trafficload(self, data, cache_size_list): qoe, userload, allload = 0, 0, 0; for batch_id in range(batch_size): #R:real data set - R = set(data[batch_id, :].unfold(-1, self.slide_T + 1, self.slide_T)[:, -1].tolist()); + R = set(data[batch_id, :].unfold(-1, self.slide_T + 1, self.slide_T + 1)[:, -1].tolist()); if len(cache_set & R) > (req_len - data[batch_id, :].eq(0).sum().item())*self.cache_satisfaction_ratio: qoe += 1; userload += len(R - cache_set); @@ -736,14 +739,6 @@ def test(self, dataset): #next_req:(batch_size, 1) _, test_data, next_req = iter(test_loader).__next__(); test_data, next_req = test_data.to(self.device), next_req.to(self.device); - #next_req_logits:(batch_size, req_types) - with torch.no_grad(): - t_r = time.time(); - next_req_logits = self.model(test_data); - t_reason += (time.time() - t_r); - #next_req:(batch_size) - next_req_pre = next_req_logits.argmax(dim = -1); - logger.debug(f'pre_types:{len(Counter(next_req_pre.tolist()))}'); #new data:(batch_size, seq_len + 1) data = torch.cat((test_data, next_req.unsqueeze(-1)), dim = -1); #calculate qoe and traffic load @@ -782,7 +777,7 @@ def _report_batch_result(self, t_start, cur_step, n_step, qoe, trafficload): def _report_result(self, result): '''Report the test result ''' - + str_show2 = '' for it in self.cache_size: str_show2 += f' {it:.1f} - {result["QoE"][it]:.6f} -- {result["TrafficLoad"][it]:.6f} \n' str = f'[{self.model.type}]Result of test\n'\ diff --git a/Room/work.py b/Room/work.py index 9f90c03..772b279 100644 --- a/Room/work.py +++ b/Room/work.py @@ -26,24 +26,30 @@ def run_work(config_path, mode = 'train'): lab_cfg = json_util.jsonload('./config/lab_cfg.json'); if mode == 'test': config = json_util.jsonload(config_path); - _, config['train']['model_save_path'] = os.path.split(config['train']['model_save_path']); - cfg_root, _ = os.path.split(config_path); - config['train']['model_save_path'] = cfg_root + '/' + config['train']['model_save_path']; - logger.info(f"Updata save path:[{config['train']['model_save_path']}]"); - json_util.jsonsave(config, config_path); - del cfg_root; + if config['net']['type'].lower() in ['fifo', 'lru', 'lfu']: + config['test']['model_save_path'] = get_save_path(config); + logger.info(f"Updata save path:[{config['test']['model_save_path']}]") + json_util.jsonsave(config, config_path); + else: + _, config['train']['model_save_path'] = os.path.split(config['train']['model_save_path']); + cfg_root, _ = os.path.split(config_path); + config['train']['model_save_path'] = cfg_root + '/' + config['train']['model_save_path']; + logger.info(f"Updata save path:[{config['train']['model_save_path']}]"); + json_util.jsonsave(config, config_path); + del cfg_root; else: config = json_util.jsonload(config_path); #set random seed torch.manual_seed(config['seed']); #initial device - if config['train']['gpu_is_available']: + if (mode == 'trian' and config['train']['gpu_is_available']) or (mode == 'test' and config['test']['gpu_is_available']): device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu"); else: device = torch.device("cpu"); + glb_var.set_value('device', device); #set training constant - if config['train']['use_amp']: + if mode == 'train' and config['train']['use_amp']: glb_var.set_value('mask_to_value', lab_cfg['constant']['use_amp_true']['mask_to_value']); glb_var.set_value('eps', lab_cfg['constant']['use_amp_true']['eps']); else: @@ -67,7 +73,7 @@ def run_work(config_path, mode = 'train'): #get model - if mode in ['train', 'train_and_test']: + if mode in ['train', 'train_and_test'] or config['net']['type'].lower() in ['fifo', 'lru', 'lfu']: model = generate_model(config); else: model = torch.load(config['train']['model_save_path']); @@ -77,10 +83,7 @@ def run_work(config_path, mode = 'train'): config['train']['model_save_path'] = get_save_path(config); logger.info(f"Updata save path:[{config['train']['model_save_path']}]") json_util.jsonsave(config, config_path); - elif mode == 'test' and config['net']['type'].lower() in ['fifo', 'lru', 'lfu']: - config['test']['model_save_path'] = get_save_path(config); - logger.info(f"Updata save path:[{config['test']['model_save_path']}]") - json_util.jsonsave(config, config_path); + report(config, lab_cfg); result = None; @@ -103,14 +106,10 @@ def run_work(config_path, mode = 'train'): tester = Tester(config, torch.load(config['train']['model_save_path'])); result = run_test(tester, dataset); elif mode == 'test': - if config['test']['gpu_is_available']: - glb_var.set_value('device', torch.device("cuda:0" if torch.cuda.is_available() else "cpu")); + if config['net']['type'].lower() in ['fifo', 'lru', 'lfu']: + tester = ConventionalTester(config, model); else: - glb_var.set_value('device', torch.device("cpu")); - if config['net']['type'].lower() in ['fifo', 'lru', 'lfu']: - tester = ConventionalTester(config, model); - else: - tester = Tester(config, model); + tester = Tester(config, model); result = run_test(tester, dataset); else: logger.error(f'Unrecognized Mode [{mode}], acceptable:(train/test/train_and_test)'); @@ -194,7 +193,7 @@ def report(config, lab_cfg): '''print the info and check config ''' #check config - keys = ['net', 'dataset', 'linux_fast_num_workers', 'email_reminder', 'seed', 'train', 'test']; + keys = ['net', 'dataset', 'linux_fast_num_workers', 'email_reminder', 'seed', 'test']; train_keys = ['batch_size', 'max_epoch', 'valid_step', 'stop_train_step_valid_not_improve', 'gpu_is_available', 'use_amp', 'optimizer_type', 'learning_rate', 'weight_decay', 'betas', 'use_lr_schedule', 'lr_max', 'metric_less', 'save', 'model_save_path', 'end_save']; @@ -228,6 +227,7 @@ def report(config, lab_cfg): except: logger.error('Please enter the config directory to configure the lab_cfg.json file'); raise callback.CustomException('ConfigError'); + logger.info( f'CacheLab Configuration report:\n' '------------------------------------\n' diff --git a/config/cl4srec/cl4srec_appliances.json b/config/cl4srec/cl4srec_appliances.json index b601cd8..a9f7f22 100644 --- a/config/cl4srec/cl4srec_appliances.json +++ b/config/cl4srec/cl4srec_appliances.json @@ -32,7 +32,7 @@ }, "seed": 5566, "linux_fast_num_workers": 4, - "email_reminder": true, + "email_reminder": false, "dataset": { "type": "appliances", "path": "./data/datasets/process/complete/Appliances.data", diff --git a/config/cl4srec/cl4srec_ml1m.json b/config/cl4srec/cl4srec_ml1m.json index 967bc10..808d2ff 100644 --- a/config/cl4srec/cl4srec_ml1m.json +++ b/config/cl4srec/cl4srec_ml1m.json @@ -32,7 +32,7 @@ }, "seed": 5566, "linux_fast_num_workers": 4, - "email_reminder": true, + "email_reminder": false, "dataset": { "type": "ml1m", "path": "./data/datasets/process/complete/ml.data", diff --git a/config/cl4srec/cl4srec_music.json b/config/cl4srec/cl4srec_music.json index feb8242..82aec2e 100644 --- a/config/cl4srec/cl4srec_music.json +++ b/config/cl4srec/cl4srec_music.json @@ -32,7 +32,7 @@ }, "seed": 5566, "linux_fast_num_workers": 4, - "email_reminder": true, + "email_reminder": false, "dataset": { "type": "music", "path": "./data/datasets/process/lite/Digital_Music_lite.data", diff --git a/config/duo4srec/duo4srec_appliances.json b/config/duo4srec/duo4srec_appliances.json index b87f610..f071f68 100644 --- a/config/duo4srec/duo4srec_appliances.json +++ b/config/duo4srec/duo4srec_appliances.json @@ -29,7 +29,7 @@ }, "seed": 5566, "linux_fast_num_workers": 4, - "email_reminder": true, + "email_reminder": false, "dataset": { "type": "appliances", "path": "./data/datasets/process/complete/Appliances.data", diff --git a/config/duo4srec/duo4srec_ml1m.json b/config/duo4srec/duo4srec_ml1m.json index d089280..347b8e1 100644 --- a/config/duo4srec/duo4srec_ml1m.json +++ b/config/duo4srec/duo4srec_ml1m.json @@ -29,7 +29,7 @@ }, "seed": 5566, "linux_fast_num_workers": 4, - "email_reminder": true, + "email_reminder": false, "dataset": { "type": "ml1m", "path": "./data/datasets/process/complete/ml.data", diff --git a/config/duo4srec/duo4srec_music.json b/config/duo4srec/duo4srec_music.json index 4f14589..66a3194 100644 --- a/config/duo4srec/duo4srec_music.json +++ b/config/duo4srec/duo4srec_music.json @@ -29,7 +29,7 @@ }, "seed": 5566, "linux_fast_num_workers": 4, - "email_reminder": true, + "email_reminder": false, "dataset": { "type": "music", "path": "./data/datasets/process/lite/Digital_Music_lite.data", diff --git a/config/ec4srec/ec4srec_appliances.json b/config/ec4srec/ec4srec_appliances.json index aea5cea..9453daf 100644 --- a/config/ec4srec/ec4srec_appliances.json +++ b/config/ec4srec/ec4srec_appliances.json @@ -38,7 +38,7 @@ }, "seed": 5566, "linux_fast_num_workers": 4, - "email_reminder": true, + "email_reminder": false, "dataset": { "type": "appliances", "path": "./data/datasets/process/complete/Appliances.data", diff --git a/config/ec4srec/ec4srec_ml1m.json b/config/ec4srec/ec4srec_ml1m.json index 34dfb2f..f21b61a 100644 --- a/config/ec4srec/ec4srec_ml1m.json +++ b/config/ec4srec/ec4srec_ml1m.json @@ -38,7 +38,7 @@ }, "seed": 5566, "linux_fast_num_workers": 4, - "email_reminder": true, + "email_reminder": false, "dataset": { "type": "ml1m", "path": "./data/datasets/process/complete/ml.data", diff --git a/config/ec4srec/ec4srec_music.json b/config/ec4srec/ec4srec_music.json index 6a10797..fb5fa48 100644 --- a/config/ec4srec/ec4srec_music.json +++ b/config/ec4srec/ec4srec_music.json @@ -38,7 +38,7 @@ }, "seed": 5566, "linux_fast_num_workers": 4, - "email_reminder": true, + "email_reminder": false, "dataset": { "type": "music", "path": "./data/datasets/process/lite/Digital_Music_lite.data", diff --git a/config/egpc/egpc_appliances.json b/config/egpc/egpc_appliances.json index 8dfa9db..37860ae 100644 --- a/config/egpc/egpc_appliances.json +++ b/config/egpc/egpc_appliances.json @@ -38,7 +38,7 @@ }, "seed": 5566, "linux_fast_num_workers": 4, - "email_reminder": true, + "email_reminder": false, "dataset": { "type": "appliances", "path": "./data/datasets/process/complete/Appliances.data", diff --git a/config/egpc/egpc_ml1m.json b/config/egpc/egpc_ml1m.json index df27f1c..f8c0f32 100644 --- a/config/egpc/egpc_ml1m.json +++ b/config/egpc/egpc_ml1m.json @@ -38,7 +38,7 @@ }, "seed": 5566, "linux_fast_num_workers": 4, - "email_reminder": true, + "email_reminder": false, "dataset": { "type": "ml1m", "path": "./data/datasets/process/complete/ml.data", diff --git a/config/egpc/egpc_music.json b/config/egpc/egpc_music.json index f2b5ca4..024e2b0 100644 --- a/config/egpc/egpc_music.json +++ b/config/egpc/egpc_music.json @@ -38,7 +38,7 @@ }, "seed": 5566, "linux_fast_num_workers": 4, - "email_reminder": true, + "email_reminder": false, "dataset": { "type": "music", "path": "./data/datasets/process/lite/Digital_Music_lite.data", diff --git a/config/fifo/fifo_appliances.json b/config/fifo/fifo_appliances.json new file mode 100644 index 0000000..26e4532 --- /dev/null +++ b/config/fifo/fifo_appliances.json @@ -0,0 +1,37 @@ +{ + "net": { + "type": "FIFO", + "bs_storagy": 1000 + }, + "seed": 6655, + "linux_fast_num_workers": 4, + "email_reminder": false, + "dataset": { + "type": "ml1m", + "path": "./data/datasets/process/complete/Appliances.data", + "crop_or_fill": true, + "fill_mask": 0, + "limit_length": 25 + }, + "test": { + "batch_size": 256, + "cache_satisfaction_ratio": 0.3, + "bs_storagy": 1000, + "slide_T": 3, + "cache_size": [ + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9, + 1.0 + ], + "gpu_is_available": false, + "save": false, + "model_save_path": "./data/saved/fifo/ml1m/model.model" + } +} \ No newline at end of file diff --git a/config/fifo/fifo_ml1m.json b/config/fifo/fifo_ml1m.json index d6752e6..89480a4 100644 --- a/config/fifo/fifo_ml1m.json +++ b/config/fifo/fifo_ml1m.json @@ -9,13 +9,13 @@ "dataset": { "type": "ml1m", "path": "./data/datasets/process/complete/ml.data", - "crop_or_fill": false, + "crop_or_fill": true, "fill_mask": 0, "limit_length": 55 }, "test": { "batch_size": 256, - "cache_satisfaction_ratio": 0.2, + "cache_satisfaction_ratio": 0.3, "bs_storagy": 1000, "slide_T": 3, "cache_size": [ @@ -31,7 +31,7 @@ 1.0 ], "gpu_is_available": false, - "save": true, - "model_save_path":"./" + "save": false, + "model_save_path": "./data/saved/fifo/ml1m/model.model" } } \ No newline at end of file diff --git a/config/fifo/fifo_music.json b/config/fifo/fifo_music.json new file mode 100644 index 0000000..1be454f --- /dev/null +++ b/config/fifo/fifo_music.json @@ -0,0 +1,37 @@ +{ + "net": { + "type": "FIFO", + "bs_storagy": 1000 + }, + "seed": 6655, + "linux_fast_num_workers": 4, + "email_reminder": false, + "dataset": { + "type": "ml1m", + "path": "./data/datasets/process/lite/Digital_Music_lite.data", + "crop_or_fill": true, + "fill_mask": 0, + "limit_length": 25 + }, + "test": { + "batch_size": 256, + "cache_satisfaction_ratio": 0.3, + "bs_storagy": 1000, + "slide_T": 3, + "cache_size": [ + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9, + 1.0 + ], + "gpu_is_available": false, + "save": false, + "model_save_path": "./data/saved/fifo/ml1m/model.model" + } +} \ No newline at end of file diff --git a/config/lfu/lfu_appliances.json b/config/lfu/lfu_appliances.json new file mode 100644 index 0000000..58fd8a1 --- /dev/null +++ b/config/lfu/lfu_appliances.json @@ -0,0 +1,38 @@ +{ + "net": { + "type": "LFU", + "bs_storagy": 1000, + "recent_used_n": 1200 + }, + "seed": 6655, + "linux_fast_num_workers": 4, + "email_reminder": false, + "dataset": { + "type": "ml1m", + "path": "./data/datasets/process/complete/Appliances.data", + "crop_or_fill": true, + "fill_mask": 0, + "limit_length": 25 + }, + "test": { + "batch_size": 256, + "cache_satisfaction_ratio": 0.4, + "bs_storagy": 1000, + "slide_T": 3, + "cache_size": [ + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9, + 1.0 + ], + "gpu_is_available": false, + "save": false, + "model_save_path": "./data/saved/lfu/ml1m/model.model" + } +} \ No newline at end of file diff --git a/config/lfu/lfu_ml1m.json b/config/lfu/lfu_ml1m.json index 87889e3..f76684d 100644 --- a/config/lfu/lfu_ml1m.json +++ b/config/lfu/lfu_ml1m.json @@ -2,50 +2,23 @@ "net": { "type": "LFU", "bs_storagy": 1000, - "recent_used_n":1200 + "recent_used_n": 1200 }, "seed": 6655, "linux_fast_num_workers": 4, "email_reminder": false, "dataset": { "type": "ml1m", - "path": "./data/datasets/process/complete/ml_devide_55.data", - "crop_or_fill": false, + "path": "./data/datasets/process/complete/ml.data", + "crop_or_fill": true, "fill_mask": 0, "limit_length": 55 }, - "train": { - "batch_size": 256, - "max_epoch": 1000, - "valid_step": 10, - "stop_train_step_valid_not_improve": 50, - "gpu_is_available": false, - "use_amp": false, - "optimizer_type": "adam", - "learning_rate": 0.001, - "weight_decay": 1e-08, - "betas": [ - 0.9, - 0.999 - ], - "use_lr_schedule": false, - "lr_max": 1e-05, - "metric_less": true, - "save": true, - "model_save_path": "./data/saved/caser/ml1m/64_8/model.model", - "end_save": false - }, "test": { "batch_size": 256, - "cache_satisfaction_ratio": 0.2, + "cache_satisfaction_ratio": 0.4, "bs_storagy": 1000, "slide_T": 3, - "alter_topk": 10, - "metrics_at_k": [ - 5, - 10, - 20 - ], "cache_size": [ 0.1, 0.2, @@ -59,6 +32,7 @@ 1.0 ], "gpu_is_available": false, - "save": true + "save": false, + "model_save_path": "./data/saved/lfu/ml1m/model.model" } } \ No newline at end of file diff --git a/config/lfu/lfu_music.json b/config/lfu/lfu_music.json new file mode 100644 index 0000000..b057b40 --- /dev/null +++ b/config/lfu/lfu_music.json @@ -0,0 +1,38 @@ +{ + "net": { + "type": "LFU", + "bs_storagy": 1000, + "recent_used_n":1200 + }, + "seed": 6655, + "linux_fast_num_workers": 4, + "email_reminder": false, + "dataset": { + "type": "ml1m", + "path": "./data/datasets/process/lite/Digital_Music_lite.data", + "crop_or_fill": true, + "fill_mask": 0, + "limit_length": 25 + }, + "test": { + "batch_size": 256, + "cache_satisfaction_ratio": 0.4, + "bs_storagy": 1000, + "slide_T": 3, + "cache_size": [ + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9, + 1.0 + ], + "gpu_is_available": false, + "save": false, + "model_save_path": "./data/saved/lru/ml1m/model.model" + } +} \ No newline at end of file diff --git a/config/lru/lru_appliances.json b/config/lru/lru_appliances.json new file mode 100644 index 0000000..46af6e9 --- /dev/null +++ b/config/lru/lru_appliances.json @@ -0,0 +1,37 @@ +{ + "net": { + "type": "LRU", + "bs_storagy": 1000 + }, + "seed": 6655, + "linux_fast_num_workers": 4, + "email_reminder": false, + "dataset": { + "type": "ml1m", + "path": "./data/datasets/process/complete/Appliances.data", + "crop_or_fill": true, + "fill_mask": 0, + "limit_length": 25 + }, + "test": { + "batch_size": 256, + "cache_satisfaction_ratio": 0.4, + "bs_storagy": 1000, + "slide_T": 3, + "cache_size": [ + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9, + 1.0 + ], + "gpu_is_available": false, + "save": false, + "model_save_path": "./data/saved/lru/ml1m/model.model" + } +} \ No newline at end of file diff --git a/config/lru/lru_ml1m.json b/config/lru/lru_ml1m.json index 3a66f0e..3bd6b1f 100644 --- a/config/lru/lru_ml1m.json +++ b/config/lru/lru_ml1m.json @@ -1,51 +1,23 @@ { "net": { "type": "LRU", - "bs_storagy": 1000, - "recent_used_n":100 + "bs_storagy": 1000 }, "seed": 6655, "linux_fast_num_workers": 4, "email_reminder": false, "dataset": { "type": "ml1m", - "path": "./data/datasets/process/complete/ml_devide_55.data", - "crop_or_fill": false, + "path": "./data/datasets/process/complete/ml.data", + "crop_or_fill": true, "fill_mask": 0, "limit_length": 55 }, - "train": { - "batch_size": 256, - "max_epoch": 1000, - "valid_step": 10, - "stop_train_step_valid_not_improve": 50, - "gpu_is_available": false, - "use_amp": false, - "optimizer_type": "adam", - "learning_rate": 0.001, - "weight_decay": 1e-08, - "betas": [ - 0.9, - 0.999 - ], - "use_lr_schedule": false, - "lr_max": 1e-05, - "metric_less": true, - "save": true, - "model_save_path": "./data/saved/caser/ml1m/64_8/model.model", - "end_save": false - }, "test": { "batch_size": 256, - "cache_satisfaction_ratio": 0.2, + "cache_satisfaction_ratio": 0.4, "bs_storagy": 1000, "slide_T": 3, - "alter_topk": 10, - "metrics_at_k": [ - 5, - 10, - 20 - ], "cache_size": [ 0.1, 0.2, @@ -59,6 +31,7 @@ 1.0 ], "gpu_is_available": false, - "save": true + "save": false, + "model_save_path": "./data/saved/lru/ml1m/model.model" } } \ No newline at end of file diff --git a/config/lru/lru_music.json b/config/lru/lru_music.json new file mode 100644 index 0000000..ec163c6 --- /dev/null +++ b/config/lru/lru_music.json @@ -0,0 +1,37 @@ +{ + "net": { + "type": "LRU", + "bs_storagy": 1000 + }, + "seed": 6655, + "linux_fast_num_workers": 4, + "email_reminder": false, + "dataset": { + "type": "ml1m", + "path": "./data/datasets/process/lite/Digital_Music_lite.data", + "crop_or_fill": true, + "fill_mask": 0, + "limit_length": 25 + }, + "test": { + "batch_size": 256, + "cache_satisfaction_ratio": 0.4, + "bs_storagy": 1000, + "slide_T": 3, + "cache_size": [ + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9, + 1.0 + ], + "gpu_is_available": false, + "save": false, + "model_save_path": "./data/saved/lru/ml1m/model.model" + } +} \ No newline at end of file diff --git a/data/generator.py b/data/generator.py index a9d5b08..717e734 100644 --- a/data/generator.py +++ b/data/generator.py @@ -181,7 +181,7 @@ def __getitem__(self, index): return self.data[index].squeeze(0).to(torch.int64); def get_dataloader(dataset, net_cfg, num_workers, loader_batch, shuffle = False, mode = 'train'): - if net_cfg['is_cl_method'] or net_cfg['type'].lower() in ['fifo', 'lru', 'lfu']: + if net_cfg['type'].lower() in ['fifo', 'lru', 'lfu'] or net_cfg['is_cl_method']: return torch.utils.data.DataLoader(NextReqDataSet(dataset, mode = mode), num_workers = num_workers, pin_memory = True, diff --git a/model/framework/lfu.py b/model/framework/lfu.py index 7bbd5cd..1246552 100644 --- a/model/framework/lfu.py +++ b/model/framework/lfu.py @@ -25,7 +25,7 @@ def generate_subcache(self, n) -> set: else: #Select the n items with the highest frequency sort_indx, c = self._sort_by_freq(); - return set(np.asarray(self.cache)[np.asarray(c)[sort_indx[-n:]]].tolist()) + return set(np.asarray(c)[sort_indx[-n:]].tolist()) def _sort_by_freq(self): '''Sort from small to large by frequence @@ -52,7 +52,7 @@ def update(self, seqs) -> None: ------ elements of seqs do not exceed the bs_storagy ''' - seqs = [seqs.tolist()]; + seqs = seqs.tolist(); n_unique = self._check_unique(seqs); if self.cache_left >= n_unique: self.cache_left -= n_unique; diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8d6e1b0 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,94 @@ +ale-py==0.8.1 +asttokens==2.2.1 +AutoROM==0.4.2 +AutoROM.accept-rom-license==0.6.1 +backcall==0.2.0 +backports.functools-lru-cache==1.6.5 +certifi==2023.7.22 +charset-normalizer==3.2.0 +click==8.1.6 +cloudpickle==2.2.1 +cmake==3.25.0 +comm==0.1.4 +contourpy==1.1.0 +cycler==0.11.0 +debugpy==1.6.7 +decorator==5.1.1 +exceptiongroup==1.1.3 +executing==1.2.0 +feedparser==6.0.11 +filelock==3.9.0 +fonttools==4.42.0 +gym==0.26.2 +gym-notices==0.0.8 +html2text==2020.1.16 +idna==3.4 +igraph==0.10.6 +importlib-metadata==6.8.0 +importlib-resources==6.0.1 +ipykernel==6.25.2 +ipython==8.14.0 +ipywidgets==8.1.0 +jedi==0.19.0 +jieba==0.42.1 +Jinja2==3.1.2 +joblib==1.3.2 +jupyter_client==8.3.1 +jupyter_core==5.3.1 +jupyterlab-widgets==3.0.8 +kiwisolver==1.4.4 +lit==15.0.7 +MarkupSafe==2.1.2 +matplotlib==3.7.2 +matplotlib-inline==0.1.6 +mpmath==1.2.1 +nest-asyncio==1.5.6 +networkx==3.1 +numpy==1.25.2 +packaging==23.1 +parso==0.8.3 +pexpect==4.8.0 +pickleshare==0.7.5 +Pillow==10.0.0 +pip==23.2.1 +platformdirs==3.10.0 +prompt-toolkit==3.0.39 +psutil==5.9.5 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pydash==7.0.6 +pyg-lib==0.2.0+pt20cu118 +Pygments==2.16.1 +pyparsing==3.0.9 +pypdf==3.17.4 +python-dateutil==2.8.2 +python-igraph==0.10.6 +pyzmq==25.1.0 +requests==2.31.0 +scikit-learn==1.3.0 +scipy==1.11.1 +setuptools==68.0.0 +sgmllib3k==1.0.0 +six==1.16.0 +stack-data==0.6.2 +sympy==1.11.1 +texttable==1.6.7 +threadpoolctl==3.2.0 +torch==2.0.1+cu118 +torch-cluster==1.6.1+pt20cu118 +torch-geometric==2.3.1 +torch-scatter==2.1.1+pt20cu118 +torch-sparse==0.6.17+pt20cu118 +torch-spline-conv==1.2.2+pt20cu118 +torchaudio==2.0.2+cu118 +torchvision==0.15.2+cu118 +tornado==6.3.2 +tqdm==4.66.1 +traitlets==5.9.0 +triton==2.0.0 +typing_extensions==4.8.0 +urllib3==2.0.4 +wcwidth==0.2.6 +wheel==0.38.4 +widgetsnbextension==4.0.8 +zipp==3.16.2