Skip to content

Commit 2284216

Browse files
CISCCISC
authored
gguf-py : add special token modification capability (ggml-org#7166)
* Add special token modification capability To be able to fix/amend special tokens in a GGUF let's add two new arguments: * `--special-token <name> <value>` where `<name>` can be bos, eos, prefix, middle, etc. while `<value>` is the token value, f.ex. `"<｜fim▁begin｜>"` * `--special-token-by-id <name> <id>` where `<id>` is the ID of the token, f.ex. 32006 So, in order to f.ex. add fill-in-middle tokens to a GGUF you would do the following: ```bash python3 gguf-new-metadata.py input.gguf output.gguf --special-token prefix "<｜fim▁begin｜>" --special-token middle "<｜fim▁hole｜>" --special-token suffix "<｜fim▁end｜>" ``` * improve help text * flake-- * fix multiple tokens warning * make script executable * switch to namedtuple, no need to dataclass * typing++ * add progress bar * Add special token modification capability To be able to fix/amend special tokens in a GGUF let's add two new arguments: * `--special-token <name> <value>` where `<name>` can be bos, eos, prefix, middle, etc. while `<value>` is the token value, f.ex. `"<｜fim▁begin｜>"` * `--special-token-by-id <name> <id>` where `<id>` is the ID of the token, f.ex. 32006 So, in order to f.ex. add fill-in-middle tokens to a GGUF you would do the following: ```bash gguf-new-metadata.py input.gguf output.gguf --special-token prefix "<｜fim▁begin｜>" --special-token middle "<｜fim▁end｜>" --special-token suffix "<｜fim▁hole｜>" ``` (yes, fim_end is the `middle` token, because completion is a `prefix`/`suffix`/`middle` sequence (where `middle` is unfilled)) or ```bash gguf-new-metadata.py input.gguf output.gguf --special-token prefix "<fim_prefix>" --special-token middle "<fim_middle>" --special-token suffix "<fim_suffix>" ``` etc... NB: The tokens have to exist already, trying to add non-existent token name/IDs will be ignored (with a warning), while non-existent values will fail (with an error). * improve help text * flake-- * fix multiple tokens warning * make script executable * switch to namedtuple, no need to dataclass * typing++ * add progress bar * fail on invalid token id
1 parent 4734524 commit 2284216

File tree

1 file changed

+72
-20
lines changed

1 file changed

+72
-20
lines changed

‎gguf-py/scripts/gguf-new-metadata.py

100644100755
Lines changed: 72 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@
77
from pathlib import Path
88

99
import numpy as np
10-
from typing import Any, Sequence
10+
from tqdm import tqdm
11+
from typing import Any, Sequence, NamedTuple
1112

1213
# Necessary to load the local gguf package
1314
if "NO_LOCAL_GGUF" not in os.environ and (Path(__file__).parent.parent.parent / 'gguf-py').exists():
@@ -18,6 +19,12 @@
1819
logger = logging.getLogger("gguf-new-metadata")
1920

2021

22+
class MetadataDetails(NamedTuple):
23+
type: gguf.GGUFValueType
24+
value: Any
25+
description: str = ''
26+
27+
2128
def get_byteorder(reader: gguf.GGUFReader) -> gguf.GGUFEndian:
2229
if np.uint32(1) == np.uint32(1).newbyteorder("<"):
2330
# Host is little endian
@@ -59,7 +66,16 @@ def get_field_data(reader: gguf.GGUFReader, key: str) -> Any:
5966
return decode_field(field)
6067

6168

62-
def copy_with_new_metadata(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, new_metadata: dict[str, str], remove_metadata: Sequence[str]) -> None:
69+
def find_token(token_list: Sequence[int], token: str) -> Sequence[int]:
70+
token_ids = [index for index, value in enumerate(token_list) if value == token]
71+
72+
if len(token_ids) == 0:
73+
raise LookupError(f'Unable to find "{token}" in token list!')
74+
75+
return token_ids
76+
77+
78+
def copy_with_new_metadata(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, new_metadata: dict[str, MetadataDetails], remove_metadata: Sequence[str]) -> None:
6379
for field in reader.fields.values():
6480
# Suppress virtual fields and fields written by GGUFWriter
6581
if field.name == gguf.Keys.General.ARCHITECTURE or field.name.startswith('GGUF.'):
@@ -75,54 +91,64 @@ def copy_with_new_metadata(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, new
7591
logger.debug(f'Removing {field.name}')
7692
continue
7793

78-
old_val = decode_field(field)
94+
old_val = MetadataDetails(field.types[0], decode_field(field))
7995
val = new_metadata.get(field.name, old_val)
8096

8197
if field.name in new_metadata:
82-
logger.debug(f'Modifying {field.name}: "{old_val}" -> "{val}"')
98+
logger.debug(f'Modifying {field.name}: "{old_val.value}" -> "{val.value}" {val.description}')
8399
del new_metadata[field.name]
84-
elif val is not None:
100+
elif val.value is not None:
85101
logger.debug(f'Copying {field.name}')
86102

87-
if val is not None:
103+
if val.value is not None:
88104
writer.add_key(field.name)
89-
writer.add_val(val, field.types[0])
105+
writer.add_val(val.value, val.type)
90106

91107
if gguf.Keys.Tokenizer.CHAT_TEMPLATE in new_metadata:
92108
logger.debug('Adding chat template(s)')
93-
writer.add_chat_template(new_metadata[gguf.Keys.Tokenizer.CHAT_TEMPLATE])
109+
writer.add_chat_template(new_metadata[gguf.Keys.Tokenizer.CHAT_TEMPLATE].value)
94110
del new_metadata[gguf.Keys.Tokenizer.CHAT_TEMPLATE]
95111

96-
# TODO: Support other types than string?
97112
for key, val in new_metadata.items():
98-
logger.debug(f'Adding {key}: {val}')
113+
logger.debug(f'Adding {key}: "{val.value}" {val.description}')
99114
writer.add_key(key)
100-
writer.add_val(val, gguf.GGUFValueType.STRING)
115+
writer.add_val(val.value, val.type)
116+
117+
total_bytes = 0
101118

102119
for tensor in reader.tensors:
120+
total_bytes += tensor.n_bytes
103121
# Dimensions are written in reverse order, so flip them first
104122
shape = np.flipud(tensor.shape).tolist()
105123
writer.add_tensor_info(tensor.name, shape, tensor.data.dtype, tensor.data.nbytes, tensor.tensor_type)
106124

125+
bar = tqdm(desc="Writing", total=total_bytes, unit="byte", unit_scale=True)
126+
107127
writer.write_header_to_file()
108128
writer.write_kv_data_to_file()
109129
writer.write_ti_data_to_file()
110130

111131
for tensor in reader.tensors:
112132
writer.write_tensor_data(tensor.data)
133+
bar.update(tensor.n_bytes)
113134

114135
writer.close()
115136

116137

117138
def main() -> None:
139+
tokenizer_metadata = (getattr(gguf.Keys.Tokenizer, n) for n in gguf.Keys.Tokenizer.__dict__.keys() if not n.startswith('_'))
140+
token_names = dict((n.split('.')[-1][:-len('_token_id')], n) for n in tokenizer_metadata if n.endswith('_token_id'))
141+
118142
parser = argparse.ArgumentParser(description="Make a copy of a GGUF file with new metadata")
119143
parser.add_argument("input", type=Path, help="GGUF format model input filename")
120144
parser.add_argument("output", type=Path, help="GGUF format model output filename")
121-
parser.add_argument("--general-name", type=str, help="The models general.name")
122-
parser.add_argument("--general-description", type=str, help="The models general.description")
123-
parser.add_argument("--chat-template", type=str, help="Chat template string (or JSON string containing templates)")
124-
parser.add_argument("--chat-template-config", type=Path, help="Config file (tokenizer_config.json) containing chat template(s)")
125-
parser.add_argument("--remove-metadata", action="append", type=str, help="Remove metadata (by key name) from output model")
145+
parser.add_argument("--general-name", type=str, help="The models general.name", metavar='"name"')
146+
parser.add_argument("--general-description", type=str, help="The models general.description", metavar='"Description ..."')
147+
parser.add_argument("--chat-template", type=str, help="Chat template string (or JSON string containing templates)", metavar='"{% ... %} ..."')
148+
parser.add_argument("--chat-template-config", type=Path, help="Config file containing chat template(s)", metavar='tokenizer_config.json')
149+
parser.add_argument("--remove-metadata", action="append", type=str, help="Remove metadata (by key name) from output model", metavar='general.url')
150+
parser.add_argument("--special-token", action="append", type=str, help="Special token by value", nargs=2, metavar=(' | '.join(token_names.keys()), '"<token>"'))
151+
parser.add_argument("--special-token-by-id", action="append", type=str, help="Special token by id", nargs=2, metavar=(' | '.join(token_names.keys()), '0'))
126152
parser.add_argument("--force", action="store_true", help="Bypass warnings without confirmation")
127153
parser.add_argument("--verbose", action="store_true", help="Increase output verbosity")
128154
args = parser.parse_args(None if len(sys.argv) > 2 else ["--help"])
@@ -133,20 +159,20 @@ def main() -> None:
133159
remove_metadata = args.remove_metadata or []
134160

135161
if args.general_name:
136-
new_metadata[gguf.Keys.General.NAME] = args.general_name
162+
new_metadata[gguf.Keys.General.NAME] = MetadataDetails(gguf.GGUFValueType.STRING, args.general_name)
137163

138164
if args.general_description:
139-
new_metadata[gguf.Keys.General.DESCRIPTION] = args.general_description
165+
new_metadata[gguf.Keys.General.DESCRIPTION] = MetadataDetails(gguf.GGUFValueType.STRING, args.general_description)
140166

141167
if args.chat_template:
142-
new_metadata[gguf.Keys.Tokenizer.CHAT_TEMPLATE] = json.loads(args.chat_template) if args.chat_template.startswith('[') else args.chat_template
168+
new_metadata[gguf.Keys.Tokenizer.CHAT_TEMPLATE] = MetadataDetails(gguf.GGUFValueType.STRING, json.loads(args.chat_template) if args.chat_template.startswith('[') else args.chat_template)
143169

144170
if args.chat_template_config:
145171
with open(args.chat_template_config, 'r') as fp:
146172
config = json.load(fp)
147173
template = config.get('chat_template')
148174
if template:
149-
new_metadata[gguf.Keys.Tokenizer.CHAT_TEMPLATE] = template
175+
new_metadata[gguf.Keys.Tokenizer.CHAT_TEMPLATE] = MetadataDetails(gguf.GGUFValueType.STRING, template)
150176

151177
if remove_metadata:
152178
logger.warning('*** Warning *** Warning *** Warning **')
@@ -166,6 +192,32 @@ def main() -> None:
166192
arch = get_field_data(reader, gguf.Keys.General.ARCHITECTURE)
167193
endianess = get_byteorder(reader)
168194

195+
token_list = get_field_data(reader, gguf.Keys.Tokenizer.LIST) or []
196+
197+
for name, token in args.special_token or []:
198+
if name not in token_names:
199+
logger.warning(f'Unknown special token "{name}", ignoring...')
200+
else:
201+
ids = find_token(token_list, token)
202+
new_metadata[token_names[name]] = MetadataDetails(gguf.GGUFValueType.UINT32, ids[0], f'= {token}')
203+
204+
if len(ids) > 1:
205+
logger.warning(f'Multiple "{token}" tokens found, choosing ID {ids[0]}, use --special-token-by-id if you want another:')
206+
logger.warning(', '.join(str(i) for i in ids))
207+
208+
for name, id_string in args.special_token_by_id or []:
209+
if name not in token_names:
210+
logger.warning(f'Unknown special token "{name}", ignoring...')
211+
elif not id_string.isdecimal():
212+
raise LookupError(f'Token ID "{id_string}" is not a valid ID!')
213+
else:
214+
id_int = int(id_string)
215+
216+
if id_int >= 0 and id_int < len(token_list):
217+
new_metadata[token_names[name]] = MetadataDetails(gguf.GGUFValueType.UINT32, id_int, f'= {token_list[id_int]}')
218+
else:
219+
raise LookupError(f'Token ID {id_int} is not within token list!')
220+
169221
if os.path.isfile(args.output) and not args.force:
170222
logger.warning('*** Warning *** Warning *** Warning **')
171223
logger.warning(f'* The "{args.output}" GGUF file already exists, it will be overwritten!')

0 commit comments

Comments
 (0)