Merge pull request #5676 from ibi-group/config-check

Optionally abort startup when unknown configuration parameters were detected

docs/Migrating-Configuration.md

@@ -0,0 +1,37 @@
+While OTP's GraphQL APIs are very, very stable even across versions, the JSON configuration schema
+is not. Changes to it are relatively frequent and you can see all PRs that change it with
+the [Github tag 'config change'](https://github.com/opentripplanner/OpenTripPlanner/pulls?q=label%3A%22config+change%22).
+
+### Migrating the JSON configuration
+
+OTP validates the configuration and prints warnings during startup. This means that when you
+migrate to a newer version, you should carefully inspect the logs. If you see messages like
+
+```
+(NodeAdapter.java:170) Unexpected config parameter: 'routingDefaults.stairsReluctance:1.65' in 'router-config.json'
+```
+
+this means there are properties in your configuration that are unknown to OTP and therefore likely
+to be a configuration error, perhaps because the schema was changed. In such a case you should
+consult the [feature](Configuration.md#otp-features), [router](RouterConfiguration.md) or 
+[build configuration documentation](BuildConfiguration.md) to find out what the new schema looks like.
+
+By default, OTP starts up even if unknown configuration parameters exist. This supports forward and backwards
+migration of config parameters independent if the OTP version. This allows the configuration to follow 
+the lifecycle of the environment and still be able to roll back OTP. 
+Combined with the config parameter substitution it also allows using the same configuration in 
+different environments. Tip! Rolling out the configuration before rolling out a new
+version of OTP, ensure you that you are safe and can roll back later. 
+
+An example: you change the location of the graphs, a critical error occurs afterwards and you need to 
+roll back. Any member of the dev-ops team should then be confident that they can roll back without 
+risk - even if the environment changed since the last OTP version was rolled out.
+
+### Aborting on invalid configuration
+
+If you want OTP to abort the startup when encountering unknown configuration parameters, you can add 
+the flag `--abortOnUnknownConfig` to your regular OTP CLI commands.
+
+This should of course be used wisely as it can also accidentally make your production instances refuse to start up.
+For some deployments this is a good solution - especially if the config substitution feature is used to inject
+environment specific information. Using this feature in the graph-build phase is less risky, than in the OTP serve phase.

mkdocs.yml

@@ -76,6 +76,7 @@ nav:
     - Router: 'RouterConfiguration.md'
     - "Route Request": 'RouteRequest.md'
     - "Realtime Updaters": 'UpdaterConfig.md'
+    - "Migrating between versions/builds": 'Migrating-Configuration.md'
 - Features explained:
     - "Routing modes": 'RoutingModes.md'
     - "In-station navigation": 'In-Station-Navigation.md'

src/main/java/org/opentripplanner/standalone/OTPMain.java

@@ -13,6 +13,7 @@
 import org.opentripplanner.raptor.configure.RaptorConfig;
 import org.opentripplanner.routing.graph.SerializedGraphObject;
 import org.opentripplanner.standalone.config.CommandLineParameters;
+import org.opentripplanner.standalone.config.ConfigModel;
 import org.opentripplanner.standalone.configure.ConstructApplication;
 import org.opentripplanner.standalone.configure.LoadApplication;
 import org.opentripplanner.standalone.server.GrizzlyServer;
@@ -113,6 +114,8 @@ private static void startOTPServer(CommandLineParameters cli) {
     var loadApp = new LoadApplication(cli);
     var config = loadApp.config();
 
+    detectUnusedConfigParams(cli, config);
+
     // Validate data sources, command line arguments and config before loading and
     // processing input data to fail early
     loadApp.validateConfigAndDataSources();
@@ -173,6 +176,15 @@ private static void startOTPServer(CommandLineParameters cli) {
     }
   }
 
+  /**
+   * Optionally, check if the config is valid and if not abort the startup process.
+   */
+  private static void detectUnusedConfigParams(CommandLineParameters cli, ConfigModel config) {
+    if (cli.abortOnUnknownConfig) {
+      config.abortOnUnknownParameters();
+    }
+  }
+
   private static void startOtpWebServer(CommandLineParameters params, ConstructApplication app) {
     // Index graph for travel search
     app.transitModel().index();

src/main/java/org/opentripplanner/standalone/config/BuildConfig.java

@@ -726,4 +726,11 @@ public int getSubwayAccessTimeSeconds() {
   public NodeAdapter asNodeAdapter() {
     return root;
   }
+
+  /**
+   * Checks if any unknown or invalid parameters were encountered while loading the configuration.
+   */
+  public boolean hasUnknownParameters() {
+    return root.hasUnknownParameters();
+  }
 }

src/main/java/org/opentripplanner/standalone/config/CommandLineParameters.java

@@ -133,6 +133,12 @@ public class CommandLineParameters {
   )
   public boolean visualize;
 
+  @Parameter(
+    names = { "--abortOnUnknownConfig" },
+    description = "Abort the startup if configuration files are found to contain unknown parameters."
+  )
+  public boolean abortOnUnknownConfig = false;
+
   /**
    * The remaining single parameter after the switches is the directory with the configuration
    * files. This directory may contain other files like the graph, input data and report files.

src/main/java/org/opentripplanner/standalone/config/ConfigModel.java

@@ -2,6 +2,7 @@
 
 import com.fasterxml.jackson.databind.node.MissingNode;
 import org.opentripplanner.framework.application.OTPFeature;
+import org.opentripplanner.framework.application.OtpAppException;
 import org.opentripplanner.framework.application.OtpFileNames;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -105,4 +106,24 @@ public static void initializeOtpFeatures(OtpConfig otpConfig) {
     OTPFeature.enableFeatures(otpConfig.otpFeatures);
     OTPFeature.logFeatureSetup();
   }
+
+  /**
+   * Checks if any unknown or invalid parameters were encountered while loading the configuration.
+   * <p>
+   * If so it throws an exception.
+   */
+  public void abortOnUnknownParameters() {
+    if (
+      (
+        otpConfig.hasUnknownParameters() ||
+        buildConfig.hasUnknownParameters() ||
+        routerConfig.hasUnknownParameters()
+      )
+    ) {
+      throw new OtpAppException(
+        "Configuration contains unknown parameters (see above for details). " +
+        "Please fix your configuration or remove --abortOnUnknownConfig from your OTP CLI parameters."
+      );
+    }
+  }
 }

src/main/java/org/opentripplanner/standalone/config/OtpConfig.java

@@ -71,4 +71,11 @@ public OtpConfig(NodeAdapter nodeAdapter, boolean logUnusedParams) {
       root.logAllWarnings(LOG::warn);
     }
   }
+
+  /**
+   * Checks if any unknown or invalid parameters were encountered while loading the configuration.
+   */
+  public boolean hasUnknownParameters() {
+    return root.hasUnknownParameters();
+  }
 }

src/main/java/org/opentripplanner/standalone/config/RouterConfig.java

@@ -150,4 +150,11 @@ public String toString() {
     // Print ONLY the values set, not default values
     return root.toPrettyString();
   }
+
+  /**
+   * Checks if any unknown or invalid parameters were encountered while loading the configuration.
+   */
+  public boolean hasUnknownParameters() {
+    return root.hasUnknownParameters();
+  }
 }

src/main/java/org/opentripplanner/standalone/config/framework/json/NodeAdapter.java

@@ -172,6 +172,13 @@ public void logAllWarnings(Consumer<String> logger) {
     allWarnings().forEach(logger);
   }
 
+  /**
+   * Checks if any unknown or invalid properties were encountered while loading the configuration.
+   */
+  public boolean hasUnknownParameters() {
+    return !unusedParams().isEmpty();
+  }
+
   /**
    * Be careful when using this method - this bypasses the NodeAdaptor, and we loose
    * track of unused parameters and cannot generate documentation for the children.

src/main/java/org/opentripplanner/standalone/config/framework/json/ParameterBuilder.java

@@ -120,10 +120,6 @@ public Boolean asBoolean(boolean defaultValue) {
     return ofOptional(BOOLEAN, defaultValue, JsonNode::asBoolean);
   }
 
-  public Map<String, Boolean> asBooleanMap() {
-    return ofOptionalMap(BOOLEAN, JsonNode::asBoolean);
-  }
-
   /** @throws OtpAppException if parameter is missing. */
   public double asDouble() {
     return ofRequired(DOUBLE).asDouble();