diff --git a/docs/api/lemonade.md b/docs/api/lemonade.md index 0c1e336ed..971230a6d 100644 --- a/docs/api/lemonade.md +++ b/docs/api/lemonade.md @@ -18,6 +18,7 @@ We have designed a set of Lemonade-specific endpoints to enable client applicati | `POST` | [`/v1/unload`](#post-v1unload) | Unload a model | | `POST` | [`/v1/audio/generations`](#post-v1audiogenerations) | Generate audio (music or sound effects) from a text prompt | | `POST` | [`/v1/3d/generations`](#post-v13dgenerations) | Generate a textured 3D mesh (GLB) from an image | +| `POST` | [`/v1/models/check-updates`](#post-v1modelscheck-updates) | Manually check downloaded models for upstream updates | | `GET` | [`/v1/models/{id}/files`](#get-v1modelsidfiles) | List resolved local file metadata for one model | | `GET` | [`/v1/health`](#get-v1health) | Check server status, such as models loaded | | `GET` | [`/v1/stats`](#get-v1stats) | Performance statistics from the last request | @@ -32,6 +33,48 @@ We have designed a set of Lemonade-specific endpoints to enable client applicati | `GET` | [`/metrics`](#get-metrics) | Prometheus metrics scrape endpoint | | `POST` | [`/internal/telemetry/flush`](#post-internaltelemetryflush) | Force-flush all queued telemetry trace spans | +## `POST /v1/models/check-updates` +![Status](https://img.shields.io/badge/status-fully_available-green) + +Explicitly checks downloaded Hugging Face-backed models for newer upstream +commits. This is the manual counterpart to the startup update check and works +even when `auto_check_model_updates=false`. + +Full offline mode remains authoritative: when `offline=true`, this endpoint +returns HTTP 409 and does not make network requests. + +### Example request + +```bash +curl -X POST http://localhost:13305/v1/models/check-updates +``` + +The same action is available from the CLI: + +```bash +lemonade check-updates +``` + +### Response format + +```json +{ + "status": "success", + "updates_available": 2, + "models": [ + "Qwen3-4B-GGUF", + "Whisper-Tiny" + ] +} +``` + +The endpoint is available at: + +- `/v1/models/check-updates` +- `/api/v1/models/check-updates` +- `/v0/models/check-updates` +- `/api/v0/models/check-updates` + ## `GET /v1/models/{id}/files` ![Status](https://img.shields.io/badge/status-fully_available-green) diff --git a/docs/guide/configuration/README.md b/docs/guide/configuration/README.md index 1306a2887..7e105dd57 100644 --- a/docs/guide/configuration/README.md +++ b/docs/guide/configuration/README.md @@ -41,6 +41,7 @@ Values set in the user's `config.json` always take precedence over these seeded "rocm_bin": "builtin", "vulkan_bin": "builtin" }, + "auto_check_model_updates": true, "cloud_providers": [], "config_version": 2, "ctx_size": -1, @@ -170,6 +171,7 @@ Values set in the user's `config.json` always take precedence over these seeded | `models_dir` | string | "auto" | Directory for cached model files. "auto" follows HF_HUB_CACHE / HF_HOME / platform default | | `ctx_size` | int | -1 | Default context size for LLM models. Use `-1` for auto-resolution: the server computes the largest context that fits in available device memory using GGUF architecture metadata. Use a positive integer to set an explicit size. | | `offline` | bool | false | Skip model downloads | +| `auto_check_model_updates` | bool | true | Check downloaded Hugging Face-backed models for updates during server startup. Set to `false` to check only with `lemonade check-updates` or `POST /v1/models/check-updates`. Manual downloads and updates remain enabled. | | `no_fetch_executables` | bool | false | Prevent downloading backend executable artifacts; backends must already be installed or use the system backend | | `disable_model_filtering` | bool | false | Show all models regardless of hardware capabilities | | `inhibit_suspend` | bool | true | Prevent the OS from suspending while inference is active. Linux only (uses systemd-logind); no-op on Windows/macOS/non-systemd environments. | diff --git a/src/cpp/cli/lemonade_client.cpp b/src/cpp/cli/lemonade_client.cpp index 517140d14..f0fe6a5d0 100644 --- a/src/cpp/cli/lemonade_client.cpp +++ b/src/cpp/cli/lemonade_client.cpp @@ -260,6 +260,44 @@ bool LemonadeClient::make_request(const std::string& path, const std::string& me throw std::runtime_error("Streaming only supports POST method"); } +int LemonadeClient::check_model_updates() const { + try { + std::string response = make_request( + "/api/v1/models/check-updates", + "POST", + "{}", + "application/json", + DEFAULT_CONNECTION_TIMEOUT_MS, + LONG_TIMEOUT_MS); + auto result = json::parse(response); + + const auto& models = result.at("models"); + if (!models.is_array()) { + throw std::runtime_error("Server returned an invalid model update response"); + } + + if (models.empty()) { + std::cout << "All downloaded models are up to date." << std::endl; + return 0; + } + + std::cout << "Updates available for " << models.size() << " model(s):" << std::endl; + for (const auto& model : models) { + if (model.is_string()) { + std::cout << " - " << model.get() << std::endl; + } + } + return 0; + } catch (const HttpError& e) { + std::cerr << "Error checking model updates: " + << extract_server_error_message(e) << std::endl; + return 1; + } catch (const std::exception& e) { + std::cerr << "Error checking model updates: " << e.what() << std::endl; + return 1; + } +} + int LemonadeClient::status(int display_port) const { try { std::string response = make_request("/api/v1/health", "GET", "", "", 500, 500); diff --git a/src/cpp/cli/main.cpp b/src/cpp/cli/main.cpp index 9ecb275d9..73cfcddf8 100644 --- a/src/cpp/cli/main.cpp +++ b/src/cpp/cli/main.cpp @@ -1209,6 +1209,8 @@ int main(int argc, char* argv[]) { // Model commands CLI::App* list_cmd = app.add_subcommand("list", "List available models. Use --downloaded to show only local models.")->group("Model management"); + CLI::App* check_updates_cmd = app.add_subcommand( + "check-updates", "Check downloaded models for upstream updates")->group("Model management"); CLI::App* pull_cmd = app.add_subcommand("pull", "Pull/download a model by registered name or Hugging Face checkpoint")->group("Model management"); CLI::App* delete_cmd = app.add_subcommand("delete", "Delete a model")->group("Model management"); @@ -1455,6 +1457,8 @@ int main(int argc, char* argv[]) { return client.status(config.port); } else if (list_cmd->count() > 0) { return client.list_models(!config.downloaded, config.list_filter); + } else if (check_updates_cmd->count() > 0) { + return client.check_model_updates(); } else if (pull_cmd->count() > 0) { if (config.model.empty()) { std::cerr << "Error: 'lemonade pull' requires a model name or Hugging Face checkpoint." << std::endl; diff --git a/src/cpp/include/lemon/model_manager.h b/src/cpp/include/lemon/model_manager.h index 1bd712d35..d2343a41d 100644 --- a/src/cpp/include/lemon/model_manager.h +++ b/src/cpp/include/lemon/model_manager.h @@ -243,12 +243,14 @@ class ModelManager { // Check if model is downloaded bool is_model_downloaded(const std::string& model_name); - // Check all downloaded models for updates on HuggingFace. + // Check all downloaded models for updates on Hugging Face. // Fetches the latest commit SHA for each model's repo and compares it // with the cached commit (refs/main). Sets update_available on models - // whose upstream repo has changed. + // whose upstream repo has changed and clears stale flags for repos that + // were successfully verified as current. Returns public model names with + // updates available. // Safe to call from a background thread — locks are internal. - void check_for_model_updates(); + std::vector check_for_model_updates(); // True if the model's backend pulls its own models on demand (e.g. flm) and // so should be skipped by the router's load-time auto-download path. @@ -367,6 +369,10 @@ class ModelManager { // (keyed by checkpoint repo). See download_registered_model. std::mutex download_locks_mutex_; std::map> download_locks_; + + // Prevent startup and manual update checks from running concurrently. + std::mutex update_check_mutex_; + mutable std::map models_cache_; mutable std::map public_model_aliases_; // public name -> canonical name mutable std::map canonical_public_names_; // canonical name -> public name diff --git a/src/cpp/include/lemon/runtime_config.h b/src/cpp/include/lemon/runtime_config.h index 2991469d6..e5527da7a 100644 --- a/src/cpp/include/lemon/runtime_config.h +++ b/src/cpp/include/lemon/runtime_config.h @@ -55,6 +55,7 @@ class RuntimeConfig { // Feature flags bool offline() const; + bool auto_check_model_updates() const; bool no_fetch_executables() const; bool disable_model_filtering() const; bool enable_dgpu_gtt() const; diff --git a/src/cpp/include/lemon/server.h b/src/cpp/include/lemon/server.h index 2f97f0c04..e45fe7567 100644 --- a/src/cpp/include/lemon/server.h +++ b/src/cpp/include/lemon/server.h @@ -102,6 +102,7 @@ class Server { void handle_live(const httplib::Request& req, httplib::Response& res); void handle_models(const httplib::Request& req, httplib::Response& res); void handle_model_by_id(const httplib::Request& req, httplib::Response& res); + void handle_model_update_check(const httplib::Request& req, httplib::Response& res); void handle_model_files(const httplib::Request& req, httplib::Response& res); void handle_chat_completions(const httplib::Request& req, httplib::Response& res); // Server-side tool-calling orchestration for Omni "collection" models. diff --git a/src/cpp/include/lemon_cli/lemonade_client.h b/src/cpp/include/lemon_cli/lemonade_client.h index 58339acbd..b0e4409b7 100644 --- a/src/cpp/include/lemon_cli/lemonade_client.h +++ b/src/cpp/include/lemon_cli/lemonade_client.h @@ -79,6 +79,7 @@ class LemonadeClient { // Model management commands int list_models(bool show_all, const std::string& name_filter = "") const; + int check_model_updates() const; // Pulls/registers a model. By default the pull is cache-first // (do_not_upgrade=true): an already-downloaded model is reused without // contacting Hugging Face. Only the explicit `lemonade pull` update flow diff --git a/src/cpp/resources/defaults.json b/src/cpp/resources/defaults.json index 1b2bf16ca..0c1163da0 100644 --- a/src/cpp/resources/defaults.json +++ b/src/cpp/resources/defaults.json @@ -6,6 +6,7 @@ "rocm_bin": "builtin", "vulkan_bin": "builtin" }, + "auto_check_model_updates": true, "cloud_providers": [], "config_version": 2, "ctx_size": -1, diff --git a/src/cpp/server/model_manager.cpp b/src/cpp/server/model_manager.cpp index ce368e213..d37defc1a 100644 --- a/src/cpp/server/model_manager.cpp +++ b/src/cpp/server/model_manager.cpp @@ -1283,11 +1283,13 @@ std::map ModelManager::get_supported_models() { return public_models; } -void ModelManager::check_for_model_updates() { +std::vector ModelManager::check_for_model_updates() { + std::lock_guard update_check_lock(update_check_mutex_); + if (auto* cfg = RuntimeConfig::global(); cfg && cfg->offline()) { LOG(DEBUG, "ModelManager") << "Offline mode enabled, skipping model update check" << std::endl; - return; + return {}; } // Collect (model_name, repo_id, cached_sha) for downloaded models, // deduplicated by repo_id so we only fetch HF API once per repo. @@ -1301,7 +1303,7 @@ void ModelManager::check_for_model_updates() { { std::lock_guard lock(models_cache_mutex_); if (!cache_valid_) { - return; + return {}; } for (const auto& [name, info] : models_cache_) { if (!info.downloaded) continue; @@ -1333,6 +1335,7 @@ void ModelManager::check_for_model_updates() { } std::unordered_set updated_models; + std::unordered_set verified_models; for (auto& [repo_id, entry] : repos) { // Skip repos with no cached SHA — can't reliably compare @@ -1356,7 +1359,15 @@ void ModelManager::check_for_model_updates() { latest_sha = model_info["sha"].get(); } - if (latest_sha.empty() || latest_sha == entry.cached_sha) continue; + if (latest_sha.empty()) continue; + + // Only clear an existing update flag after a successful response + // with a usable upstream SHA. Transient failures must not hide a + // previously discovered update. + for (const auto& model_name : entry.model_names) { + verified_models.insert(model_name); + } + if (latest_sha == entry.cached_sha) continue; LOG(INFO, "ModelManager") << "Update available for " << repo_id << ": cached=" << entry.cached_sha.substr(0, 12) @@ -1372,16 +1383,31 @@ void ModelManager::check_for_model_updates() { } } - if (!updated_models.empty()) { + std::vector public_updated_models; + { std::lock_guard lock(models_cache_mutex_); for (auto& [name, info] : models_cache_) { - if (updated_models.count(name)) { - info.update_available = true; + if (verified_models.count(name)) { + info.update_available = updated_models.count(name) != 0; } } + + public_updated_models.reserve(updated_models.size()); + for (const auto& name : updated_models) { + auto public_it = canonical_public_names_.find(name); + public_updated_models.push_back( + public_it != canonical_public_names_.end() ? public_it->second : name); + } + } + + std::sort(public_updated_models.begin(), public_updated_models.end()); + + if (!public_updated_models.empty()) { LOG(INFO, "ModelManager") << "Updates available for " << updated_models.size() << " model(s)" << std::endl; } + + return public_updated_models; } static void load_checkpoints(ModelInfo& info, json& model_json) { diff --git a/src/cpp/server/runtime_config.cpp b/src/cpp/server/runtime_config.cpp index e19e10708..e1c6b83b0 100644 --- a/src/cpp/server/runtime_config.cpp +++ b/src/cpp/server/runtime_config.cpp @@ -276,6 +276,11 @@ bool RuntimeConfig::offline() const { return config_["offline"].get(); } +bool RuntimeConfig::auto_check_model_updates() const { + std::shared_lock lock(mutex_); + return config_.value("auto_check_model_updates", true); +} + bool RuntimeConfig::no_fetch_executables() const { std::shared_lock lock(mutex_); return config_["no_fetch_executables"].get(); @@ -533,6 +538,7 @@ void RuntimeConfig::validate(const std::string& key, const json& value) const { throw std::invalid_argument("'" + key + "' must be a string"); } } else if (key == "no_broadcast" || key == "offline" || + key == "auto_check_model_updates" || key == "no_fetch_executables" || key == "disable_model_filtering" || key == "enable_dgpu_gtt") { if (!value.is_boolean()) { diff --git a/src/cpp/server/server.cpp b/src/cpp/server/server.cpp index f2c4f3aec..b21f0689d 100644 --- a/src/cpp/server/server.cpp +++ b/src/cpp/server/server.cpp @@ -445,15 +445,21 @@ void Server::start_model_cache_warmup() { LOG(WARNING, "Server") << "Model list cache warmup failed with unknown error" << std::endl; } - try { - LOG(DEBUG, "Server") << "Checking downloaded models for updates..." << std::endl; - model_manager_->check_for_model_updates(); - LOG(DEBUG, "Server") << "Model update check complete" << std::endl; - } catch (const std::exception& e) { - LOG(WARNING, "Server") << "Model update check failed: " << e.what() << std::endl; - } catch (...) { - LOG(WARNING, "Server") << "Model update check failed with unknown error" << std::endl; + if (config_->auto_check_model_updates()) { + try { + LOG(DEBUG, "Server") << "Checking downloaded models for updates..." << std::endl; + (void)model_manager_->check_for_model_updates(); + LOG(DEBUG, "Server") << "Model update check complete" << std::endl; + } catch (const std::exception& e) { + LOG(WARNING, "Server") << "Model update check failed: " << e.what() << std::endl; + } catch (...) { + LOG(WARNING, "Server") << "Model update check failed with unknown error" << std::endl; + } + } else { + LOG(DEBUG, "Server") + << "Automatic model update checks are disabled" << std::endl; } + update_check_done_ = true; }); } @@ -697,6 +703,11 @@ void Server::setup_routes(httplib::Server &web_server) { handle_models(req, res); }); + // Explicit network action for users who disable startup update checks. + register_post("models/check-updates", [this](const httplib::Request& req, httplib::Response& res) { + handle_model_update_check(req, res); + }); + // Model files endpoint for the Files tab. Register before the generic // /models/(.+) route so '/files' is not parsed as the model ID. web_server.Get(R"(/api/v0/models/(.+)/files)", [this](const httplib::Request& req, httplib::Response& res) { @@ -1999,6 +2010,36 @@ void Server::handle_live(const httplib::Request& req, httplib::Response& res) { res.status = 200; } +void Server::handle_model_update_check(const httplib::Request& req, httplib::Response& res) { + (void)req; + + // A manual check is intentionally independent from + // auto_check_model_updates, but full offline mode remains authoritative. + if (config_->offline()) { + res.status = 409; + res.set_content( + nlohmann::json{{"error", "Cannot check model updates while offline=true"}}.dump(), + "application/json"); + return; + } + + try { + auto updated_models = model_manager_->check_for_model_updates(); + nlohmann::json response = { + {"status", "success"}, + {"updates_available", updated_models.size()}, + {"models", updated_models} + }; + res.set_content(response.dump(), "application/json"); + } catch (const std::exception& e) { + LOG(WARNING, "Server") << "Manual model update check failed: " << e.what() << std::endl; + res.status = 500; + res.set_content( + nlohmann::json{{"error", std::string("Model update check failed: ") + e.what()}}.dump(), + "application/json"); + } +} + void Server::handle_models(const httplib::Request& req, httplib::Response& res) { // For HEAD requests, just return 200 OK without processing if (req.method == "HEAD") {