Skip to content

Commit

Permalink
[2024.01.19 CoCheLab] New Feature(#3)
Browse files Browse the repository at this point in the history 
1.FIFO/LFU/LRU is available.
2.Fixed some bugs with LFU in the initial version.
  • Loading branch information
@DarriusL
DarriusL committed Jan 19, 2024
1 parent 7a7b817 commit 6c65c65
Show file tree
Hide file tree
Showing 28 changed files with 477 additions and 148 deletions.
1 change: 0 additions & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
*.pyc
*.log
*.data
*.txt

#Data
__pycache__/
Expand Down
120 changes: 95 additions & 25 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,32 +6,73 @@ Code for the **Co**ntent Ca**ch**ing algorithm in edge caching.

```shell
git clone https://github.com/DarriusL/CacheLab.git
```

Create an environment using one of these methods:

### conda

```shell
cd CacheLab
conda env create -f cachelab_dev.yml
conda activate cachelab_dev
git update-index --assume-unchanged config/lab_cfg.json
git update-index --assume-unchanged config/data_process_cfg.json
```

### pip

```shell
conda create -n CacheLab python=3.11
pip install -r requirements.txt
```



## Framework file structure

```
CacheLab
├── .gitignore
├── cache
│ └── logger
│ └── logger.log
├── cachelab_env.yml
├── config
│ ├── caser/.
│ ├── caser
│ │ ├── caser_appliances.json
│ │ ├── caser_ml1m.json
│ │ └── caser_music.json
│ ├── CFG_README.md
│ ├── cl4srec/.
│ ├── cl4srec
│ │ ├── cl4srec_appliances.json
│ │ ├── cl4srec_ml1m.json
│ │ └── cl4srec_music.json
│ ├── data_process_cfg.json
│ ├── duo4srec/.
│ ├── ec4srec/.
│ ├── egpc/.
│ ├── duo4srec
│ │ ├── duo4srec_appliances.json
│ │ ├── duo4srec_ml1m.json
│ │ └── duo4srec_music.json
│ ├── ec4srec
│ │ ├── ec4srec_appliances.json
│ │ ├── ec4srec_ml1m.json
│ │ └── ec4srec_music.json
│ ├── egpc
│ │ ├── egpc_appliances.json
│ │ ├── egpc_ml1m.json
│ │ └── egpc_music.json
│ ├── fifo
│ │ ├── fifo_appliances.json
│ │ ├── fifo_ml1m.json
│ │ └── fifo_music.json
│ ├── lab_cfg.json
│ └── psac/.
│ ├── lfu
│ │ ├── lfu_appliances.json
│ │ ├── lfu_ml1m.json
│ │ └── lfu_music.json
│ ├── lru
│ │ ├── lru_appliances.json
│ │ ├── lru_ml1m.json
│ │ └── lru_music.json
│ └── psac
│ ├── psac_gen_appliances.json
│ ├── psac_gen_ml1m.json
│ └── psac_gen_music.json
├── data
│ ├── augmentation.py
│ ├── datasets
Expand Down Expand Up @@ -73,28 +114,33 @@ CacheLab
│ │ └── music_devide_25.data
│ ├── generator.py
│ ├── processor.py
│ ├── saved/.
│ ├── saved./
│ └── __init__.py
|
├── executor.py
├── lib
│ ├── callback.py
│ ├── glb_var.py
│ ├── graph_util.py
│ ├── json_util.py
│ └── util.py
|
├── LICENSE
├── model
│ ├── attnet.py
│ ├── cnnnet.py
│ └── framework
│ ├── caser.py
│ ├── cl4srec.py
│ ├── duo4srec.py
│ ├── ec4srec.py
│ ├── egpc.py
│ └── psac.py
│ ├── framework
│ │ ├── base.py
│ │ ├── caser.py
│ │ ├── cl4srec.py
│ │ ├── duo4srec.py
│ │ ├── ec4srec.py
│ │ ├── egpc.py
│ │ ├── fifo.py
│ │ ├── lfu.py
│ │ ├── lru.py
│ │ └── psac.py
│ ├── loss.py
│ └── __init__.py
├── README.md
├── requirements.txt
└── Room
├── officer.py
├── work.py
Expand Down Expand Up @@ -126,9 +172,9 @@ If you need to use the provided processed data set, download it to path : ./data

Coventional:

- [ ] FIFO
- [ ] LRU
- [ ] LFU
- [x] FIFO
- [x] LRU
- [x] LFU

CL-based:

Expand Down Expand Up @@ -181,6 +227,30 @@ You need to configure the data processing configuration file yourself:@./config/
python executor.py --data_process=True
```

FIFO

```shell
python executor.py -sc='./config/fifo/fifo_ml1m.json' --mode=test
python executor.py -sc='./config/fifo/fifo_appliances.json' --mode=test
python executor.py -sc='./config/fifo/fifo_music.json' --mode=test
```

LRU

```shell
python executor.py -sc='./config/lru/lru_ml1m.json' --mode=test
python executor.py -sc='./config/lru/lru_appliances.json' --mode=test
python executor.py -sc='./config/lru/lru_music.json' --mode=test
```

LFU

```shell
python executor.py -sc='./config/lfu/lfu_ml1m.json' --mode=test
python executor.py -sc='./config/lfu/lfu_appliances.json' --mode=test
python executor.py -sc='./config/lfu/lfu_music.json' --mode=test
```

CL4SRec

```shell
Expand Down
29 changes: 12 additions & 17 deletions Room/officer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ def get_save_path(cfg):
return './data/saved/' + cfg['net']['type'].lower() + '/' + cfg['dataset']['type'] + '/' + f'{cfg["net"]["d"]}_{cfg["net"]["n_kernels"]}/model.model';
elif cfg['net']['type'].lower() in ['fifo','lru','lfu']:
return './data/saved/' + cfg['net']['type'].lower() + '/' + cfg['dataset']['type'] + '/model.model';
elif cfg['net']['is_norm_fist']:
elif cfg['net']['is_norm_first']:
norm_type = 'pre';
else:
norm_type = 'post';
Expand Down Expand Up @@ -88,7 +88,11 @@ class AbstractTester():
def __init__(self, test_cfg_dict, model) -> None:
util.set_attr(self, test_cfg_dict);
self.device = glb_var.get_value('device');
self.model = model.to(self.device);
if model.type.lower() not in ['fifo', 'lru', 'lfu']:
self.model = model.to(self.device);
else:
self.model = model;


def test(self, test_data):
logger.error('Method needs to be called after being implemented');
Expand Down Expand Up @@ -644,16 +648,15 @@ def _report_result(self, result):
logger.info(str);
return str;

class ConventionalTester():
class ConventionalTester(AbstractTester):
'''Tester for conventional algorithm: FIFO,LRU,LFU
Parameters:
-----------
'''
def __init__(self, config, model) -> None:
util.set_attr(self, config['test']);
self.model = model;
super().__init__(config['test'], model);
self.cfg = config;
if self.save:
self.save_path, _ = os.path.split(self.model_save_path);
Expand Down Expand Up @@ -683,8 +686,8 @@ def _caching_and_cal_qoe_trafficload(self, data, cache_size_list):
self.model.clear();
for batch_id in range(batch_size):
#su:(slide_len, T)
su = data[batch_id, :].unfold(-1, self.slide_T + 1, self.slide_T)[:, :self.slide_T];
self.model.update(su.reshape(1, -1));
su = data[batch_id, :].unfold(-1, self.slide_T + 1, self.slide_T + 1)[:, :self.slide_T];
self.model.update(su.reshape(-1));
QoE = {};
TrafficLoad = {};
for cache_size in cache_size_list:
Expand All @@ -697,7 +700,7 @@ def _caching_and_cal_qoe_trafficload(self, data, cache_size_list):
qoe, userload, allload = 0, 0, 0;
for batch_id in range(batch_size):
#R:real data set
R = set(data[batch_id, :].unfold(-1, self.slide_T + 1, self.slide_T)[:, -1].tolist());
R = set(data[batch_id, :].unfold(-1, self.slide_T + 1, self.slide_T + 1)[:, -1].tolist());
if len(cache_set & R) > (req_len - data[batch_id, :].eq(0).sum().item())*self.cache_satisfaction_ratio:
qoe += 1;
userload += len(R - cache_set);
Expand Down Expand Up @@ -736,14 +739,6 @@ def test(self, dataset):
#next_req:(batch_size, 1)
_, test_data, next_req = iter(test_loader).__next__();
test_data, next_req = test_data.to(self.device), next_req.to(self.device);
#next_req_logits:(batch_size, req_types)
with torch.no_grad():
t_r = time.time();
next_req_logits = self.model(test_data);
t_reason += (time.time() - t_r);
#next_req:(batch_size)
next_req_pre = next_req_logits.argmax(dim = -1);
logger.debug(f'pre_types:{len(Counter(next_req_pre.tolist()))}');
#new data:(batch_size, seq_len + 1)
data = torch.cat((test_data, next_req.unsqueeze(-1)), dim = -1);
#calculate qoe and traffic load
Expand Down Expand Up @@ -782,7 +777,7 @@ def _report_batch_result(self, t_start, cur_step, n_step, qoe, trafficload):
def _report_result(self, result):
'''Report the test result
'''

str_show2 = ''
for it in self.cache_size:
str_show2 += f' {it:.1f} - {result["QoE"][it]:.6f} -- {result["TrafficLoad"][it]:.6f} \n'
str = f'[{self.model.type}]Result of test\n'\
Expand Down
42 changes: 21 additions & 21 deletions Room/work.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,24 +26,30 @@ def run_work(config_path, mode = 'train'):
lab_cfg = json_util.jsonload('./config/lab_cfg.json');
if mode == 'test':
config = json_util.jsonload(config_path);
_, config['train']['model_save_path'] = os.path.split(config['train']['model_save_path']);
cfg_root, _ = os.path.split(config_path);
config['train']['model_save_path'] = cfg_root + '/' + config['train']['model_save_path'];
logger.info(f"Updata save path:[{config['train']['model_save_path']}]");
json_util.jsonsave(config, config_path);
del cfg_root;
if config['net']['type'].lower() in ['fifo', 'lru', 'lfu']:
config['test']['model_save_path'] = get_save_path(config);
logger.info(f"Updata save path:[{config['test']['model_save_path']}]")
json_util.jsonsave(config, config_path);
else:
_, config['train']['model_save_path'] = os.path.split(config['train']['model_save_path']);
cfg_root, _ = os.path.split(config_path);
config['train']['model_save_path'] = cfg_root + '/' + config['train']['model_save_path'];
logger.info(f"Updata save path:[{config['train']['model_save_path']}]");
json_util.jsonsave(config, config_path);
del cfg_root;
else:
config = json_util.jsonload(config_path);
#set random seed
torch.manual_seed(config['seed']);
#initial device
if config['train']['gpu_is_available']:
if (mode == 'trian' and config['train']['gpu_is_available']) or (mode == 'test' and config['test']['gpu_is_available']):
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu");
else:
device = torch.device("cpu");

glb_var.set_value('device', device);
#set training constant
if config['train']['use_amp']:
if mode == 'train' and config['train']['use_amp']:
glb_var.set_value('mask_to_value', lab_cfg['constant']['use_amp_true']['mask_to_value']);
glb_var.set_value('eps', lab_cfg['constant']['use_amp_true']['eps']);
else:
Expand All @@ -67,7 +73,7 @@ def run_work(config_path, mode = 'train'):


#get model
if mode in ['train', 'train_and_test']:
if mode in ['train', 'train_and_test'] or config['net']['type'].lower() in ['fifo', 'lru', 'lfu']:
model = generate_model(config);
else:
model = torch.load(config['train']['model_save_path']);
Expand All @@ -77,10 +83,7 @@ def run_work(config_path, mode = 'train'):
config['train']['model_save_path'] = get_save_path(config);
logger.info(f"Updata save path:[{config['train']['model_save_path']}]")
json_util.jsonsave(config, config_path);
elif mode == 'test' and config['net']['type'].lower() in ['fifo', 'lru', 'lfu']:
config['test']['model_save_path'] = get_save_path(config);
logger.info(f"Updata save path:[{config['test']['model_save_path']}]")
json_util.jsonsave(config, config_path);


report(config, lab_cfg);
result = None;
Expand All @@ -103,14 +106,10 @@ def run_work(config_path, mode = 'train'):
tester = Tester(config, torch.load(config['train']['model_save_path']));
result = run_test(tester, dataset);
elif mode == 'test':
if config['test']['gpu_is_available']:
glb_var.set_value('device', torch.device("cuda:0" if torch.cuda.is_available() else "cpu"));
if config['net']['type'].lower() in ['fifo', 'lru', 'lfu']:
tester = ConventionalTester(config, model);
else:
glb_var.set_value('device', torch.device("cpu"));
if config['net']['type'].lower() in ['fifo', 'lru', 'lfu']:
tester = ConventionalTester(config, model);
else:
tester = Tester(config, model);
tester = Tester(config, model);
result = run_test(tester, dataset);
else:
logger.error(f'Unrecognized Mode [{mode}], acceptable:(train/test/train_and_test)');
Expand Down Expand Up @@ -194,7 +193,7 @@ def report(config, lab_cfg):
'''print the info and check config
'''
#check config
keys = ['net', 'dataset', 'linux_fast_num_workers', 'email_reminder', 'seed', 'train', 'test'];
keys = ['net', 'dataset', 'linux_fast_num_workers', 'email_reminder', 'seed', 'test'];
train_keys = ['batch_size', 'max_epoch', 'valid_step', 'stop_train_step_valid_not_improve', 'gpu_is_available', 'use_amp',
'optimizer_type', 'learning_rate', 'weight_decay', 'betas', 'use_lr_schedule', 'lr_max', 'metric_less',
'save', 'model_save_path', 'end_save'];
Expand Down Expand Up @@ -228,6 +227,7 @@ def report(config, lab_cfg):
except:
logger.error('Please enter the config directory to configure the lab_cfg.json file');
raise callback.CustomException('ConfigError');

logger.info(
f'CacheLab Configuration report:\n'
'------------------------------------\n'
Expand Down
2 changes: 1 addition & 1 deletion config/cl4srec/cl4srec_appliances.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
},
"seed": 5566,
"linux_fast_num_workers": 4,
"email_reminder": true,
"email_reminder": false,
"dataset": {
"type": "appliances",
"path": "./data/datasets/process/complete/Appliances.data",
Expand Down
2 changes: 1 addition & 1 deletion config/cl4srec/cl4srec_ml1m.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
},
"seed": 5566,
"linux_fast_num_workers": 4,
"email_reminder": true,
"email_reminder": false,
"dataset": {
"type": "ml1m",
"path": "./data/datasets/process/complete/ml.data",
Expand Down
2 changes: 1 addition & 1 deletion config/cl4srec/cl4srec_music.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
},
"seed": 5566,
"linux_fast_num_workers": 4,
"email_reminder": true,
"email_reminder": false,
"dataset": {
"type": "music",
"path": "./data/datasets/process/lite/Digital_Music_lite.data",
Expand Down
2 changes: 1 addition & 1 deletion config/duo4srec/duo4srec_appliances.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
},
"seed": 5566,
"linux_fast_num_workers": 4,
"email_reminder": true,
"email_reminder": false,
"dataset": {
"type": "appliances",
"path": "./data/datasets/process/complete/Appliances.data",
Expand Down
Loading

0 comments on commit 6c65c65

Please sign in to comment.