[GuideLLM Refactor] Core: Reintroduce changes from main (#364)

<!--
Include a short paragraph of the changes introduced in this PR.
If this PR requires additional context or rationale, explain why
the changes are necessary.
-->

TODO

---

- [x] "I certify that all code in this PR is my own, except as noted
below."

- [ ] Includes AI-assisted code completion
- [ ] Includes code generated by an AI application
- [ ] Includes AI-generated tests (NOTE: AI written tests should have a
docstring that includes `## WRITTEN BY AI ##`)

Author: sjmonson

pyproject.toml

@@ -10,6 +10,9 @@ include = ["*"]
 [tool.setuptools.package-data]
 "guidellm.data" = ["*.gz"]
 
+[tool.pdm]
+distribution = true
+
 
 # ************************************************
 # ********** Project Metadata **********
@@ -70,6 +73,10 @@ perf = [
     "msgspec",
     "uvloop",
 ]
+recommended = [
+    "tiktoken>=0.11.0",  # For OpenAI tokenizer
+    "blobfile>=3.1.0",   # For OpenAI tokenizer
+]
 dev = [
     # build
     "build>=1.0.0",
@@ -110,6 +117,9 @@ dev = [
     "mkdocs-linkcheck~=1.0.6",
 ]
 
+[dependency-groups]
+dev = [ "guidellm[dev]" ]
+
 [project.urls]
 homepage = "https://github.com/vllm-project/guidellm"
 source = "https://github.com/vllm-project/guidellm"