Merge remote-tracking branch 'upstream/master' into boardd-no-fault

cereal/README.md

@@ -29,6 +29,50 @@ then means breaking backwards-compatibility with all old logs of your fork. So w
 [custom.capnp](custom.capnp) that we will leave empty in mainline cereal/openpilot. **If you only modify those, you can ensure your
 fork will remain backwards-compatible with all versions of mainline openpilot and your fork.**
 
+An example of compatible changes:
+```diff
+diff --git a/cereal/custom.capnp b/cereal/custom.capnp
+index 3348e859e..3365c7b98 100644
+--- a/cereal/custom.capnp
++++ b/cereal/custom.capnp
+@@ -10,7 +10,11 @@ $Cxx.namespace("cereal");
+ # DO rename the structs
+ # DON'T change the identifier (e.g. @0x81c2f05a394cf4af)
+
+-struct CustomReserved0 @0x81c2f05a394cf4af {
++struct SteeringInfo @0x81c2f05a394cf4af {
++  active @0 :Bool;
++  steeringAngleDeg @1 :Float32;
++  steeringRateDeg @2 :Float32;
++  steeringAccelDeg @3 :Float32;
+ }
+
+ struct CustomReserved1 @0xaedffd8f31e7b55d {
+diff --git a/cereal/log.capnp b/cereal/log.capnp
+index 1209f3fd9..b189f58b6 100644
+--- a/cereal/log.capnp
++++ b/cereal/log.capnp
+@@ -2558,14 +2558,14 @@ struct Event {
+
+     # DO change the name of the field
+     # DON'T change anything after the "@"
+-    customReservedRawData0 @124 :Data;
++    rawCanData @124 :Data;
+     customReservedRawData1 @125 :Data;
+     customReservedRawData2 @126 :Data;
+
+     # DO change the name of the field and struct
+     # DON'T change the ID (e.g. @107)
+     # DON'T change which struct it points to
+-    customReserved0 @107 :Custom.CustomReserved0;
++    steeringInfo @107 :Custom.SteeringInfo;
+     customReserved1 @108 :Custom.CustomReserved1;
+     customReserved2 @109 :Custom.CustomReserved2;
+     customReserved3 @110 :Custom.CustomReserved3;
+```
+
+---
+
 Example
 ---
 ```python

cereal/custom.capnp

@@ -7,7 +7,9 @@ $Cxx.namespace("cereal");
 # These structs are guaranteed to remain reserved and empty in mainline
 # cereal, so use these if you want custom events in your fork.
 
-# you can rename the struct, but don't change the identifier
+# DO rename the structs
+# DON'T change the identifier (e.g. @0x81c2f05a394cf4af)
+
 struct CustomReserved0 @0x81c2f05a394cf4af {
 }
 
@@ -37,3 +39,33 @@ struct CustomReserved8 @0xf416ec09499d9d19 {
 
 struct CustomReserved9 @0xa1680744031fdb2d {
 }
+
+struct CustomReserved10 @0xcb9fd56c7057593a {
+}
+
+struct CustomReserved11 @0xc2243c65e0340384 {
+}
+
+struct CustomReserved12 @0x9ccdc8676701b412 {
+}
+
+struct CustomReserved13 @0xcd96dafb67a082d0 {
+}
+
+struct CustomReserved14 @0xb057204d7deadf3f {
+}
+
+struct CustomReserved15 @0xbd443b539493bc68 {
+}
+
+struct CustomReserved16 @0xfc6241ed8877b611 {
+}
+
+struct CustomReserved17 @0xa30662f84033036c {
+}
+
+struct CustomReserved18 @0xc86a3d38d13eb3ef {
+}
+
+struct CustomReserved19 @0xa4f1eb3323f5f582 {
+}

cereal/log.capnp

@@ -2554,11 +2554,17 @@ struct Event {
     livestreamWideRoadEncodeData @121 :EncodeData;
     livestreamDriverEncodeData @122 :EncodeData;
 
+    # *********** Custom: reserved for forks ***********
+
+    # DO change the name of the field
+    # DON'T change anything after the "@"
     customReservedRawData0 @124 :Data;
     customReservedRawData1 @125 :Data;
     customReservedRawData2 @126 :Data;
 
-    # *********** Custom: reserved for forks ***********
+    # DO change the name of the field and struct
+    # DON'T change the ID (e.g. @107)
+    # DON'T change which struct it points to 
     customReserved0 @107 :Custom.CustomReserved0;
     customReserved1 @108 :Custom.CustomReserved1;
     customReserved2 @109 :Custom.CustomReserved2;
@@ -2569,6 +2575,16 @@ struct Event {
     customReserved7 @114 :Custom.CustomReserved7;
     customReserved8 @115 :Custom.CustomReserved8;
     customReserved9 @116 :Custom.CustomReserved9;
+    customReserved10 @136 :Custom.CustomReserved10;
+    customReserved11 @137 :Custom.CustomReserved11;
+    customReserved12 @138 :Custom.CustomReserved12;
+    customReserved13 @139 :Custom.CustomReserved13;
+    customReserved14 @140 :Custom.CustomReserved14;
+    customReserved15 @141 :Custom.CustomReserved15;
+    customReserved16 @142 :Custom.CustomReserved16;
+    customReserved17 @143 :Custom.CustomReserved17;
+    customReserved18 @144 :Custom.CustomReserved18;
+    customReserved19 @145 :Custom.CustomReserved19;
 
     # *********** legacy + deprecated ***********
     model @9 :Legacy.ModelData; # TODO: rename modelV2 and mark this as deprecated

common/logging_extra.py

@@ -8,13 +8,16 @@
 import socket
 import logging
 import traceback
+import numpy as np
 from threading import local
 from collections import OrderedDict
 from contextlib import contextmanager
 
 LOG_TIMESTAMPS = "LOG_TIMESTAMPS" in os.environ
 
 def json_handler(obj):
+  if isinstance(obj, np.bool_):
+    return bool(obj)
   # if isinstance(obj, (datetime.date, datetime.time)):
   #   return obj.isoformat()
   return repr(obj)

common/util.cc

@@ -294,4 +294,17 @@ std::string check_output(const std::string& command) {
   return result;
 }
 
+bool system_time_valid() {
+  // Default to August 26, 2024
+  tm min_tm = {.tm_year = 2024 - 1900, .tm_mon = 7, .tm_mday = 26};
+  time_t min_date = mktime(&min_tm);
+
+  struct stat st;
+  if (stat("/lib/systemd/systemd", &st) == 0) {
+    min_date = std::max(min_date, st.st_mtime + 86400);  // Add 1 day (86400 seconds)
+  }
+
+  return time(nullptr) > min_date;
+}
+
 }  // namespace util

common/util.h

@@ -96,6 +96,8 @@ bool create_directories(const std::string &dir, mode_t mode);
 
 std::string check_output(const std::string& command);
 
+bool system_time_valid();
+
 inline void sleep_for(const int milliseconds) {
   if (milliseconds > 0) {
     std::this_thread::sleep_for(std::chrono::milliseconds(milliseconds));

docs/CONTRIBUTING.md

@@ -63,3 +63,12 @@ A good pull request has all of the following:
 * Connect your device to Wi-Fi regularly, so that we can pull data for training better driving models.
 * Run the `nightly` branch and report issues. This branch is like `master` but it's built just like a release.
 * Annotate images in the [comma10k dataset](https://github.com/commaai/comma10k).
+
+## Contributing Training Data
+
+### A guide for forks
+
+In order for your fork's data to be eligible for the training set:
+* **Your cereal messaging structs must be [compatible](../cereal#custom-forks)**
+* **The definitions of all the stock messaging structs must not change**: Do not change how any of the fields are set, including everything from `selfdriveState.enabled` to `carState.steeringAngleDeg`. Instead, create your own structs and set them however you'd like.
+* **Do not include cars that are not supported in upstream platforms**: Instead, create new opendbc platforms for cars that you'd like to support outside of upstream, even if it's just a trim-level difference.

selfdrive/car/card.py

@@ -125,6 +125,15 @@ def __init__(self, CI=None, RI=None) -> None:
       self.CP.safetyConfigs = [safety_config]
 
     if self.CP.secOcRequired and not self.params.get_bool("IsReleaseBranch"):
+      # Copy user key if available
+      try:
+        with open("/cache/params/SecOCKey") as f:
+          user_key = f.readline().strip()
+          if len(user_key) == 32:
+            self.params.put("SecOCKey", user_key)
+      except Exception:
+        pass
+
       secoc_key = self.params.get("SecOCKey", encoding='utf8')
       if secoc_key is not None:
         saved_secoc_key = bytes.fromhex(secoc_key.strip())

selfdrive/modeld/fill_model_msg.py

@@ -71,9 +71,7 @@ def fill_model_msg(base_msg: capnp._DynamicStructBuilder, extended_msg: capnp._D
   driving_model_data.frameIdExtra = vipc_frame_id_extra
   driving_model_data.frameDropPerc = frame_drop_perc
   driving_model_data.modelExecutionTime = model_execution_time
-
-  action = driving_model_data.action
-  action.desiredCurvature = float(net_output_data['desired_curvature'][0,0])
+  driving_model_data.action.desiredCurvature = float(net_output_data['desired_curvature'][0,0])
 
   modelV2 = extended_msg.modelV2
   modelV2.frameId = vipc_frame_id
@@ -84,16 +82,11 @@ def fill_model_msg(base_msg: capnp._DynamicStructBuilder, extended_msg: capnp._D
   modelV2.modelExecutionTime = model_execution_time
 
   # plan
-  position = modelV2.position
-  fill_xyzt(position, ModelConstants.T_IDXS, *net_output_data['plan'][0,:,Plan.POSITION].T, *net_output_data['plan_stds'][0,:,Plan.POSITION].T)
-  velocity = modelV2.velocity
-  fill_xyzt(velocity, ModelConstants.T_IDXS, *net_output_data['plan'][0,:,Plan.VELOCITY].T)
-  acceleration = modelV2.acceleration
-  fill_xyzt(acceleration, ModelConstants.T_IDXS, *net_output_data['plan'][0,:,Plan.ACCELERATION].T)
-  orientation = modelV2.orientation
-  fill_xyzt(orientation, ModelConstants.T_IDXS, *net_output_data['plan'][0,:,Plan.T_FROM_CURRENT_EULER].T)
-  orientation_rate = modelV2.orientationRate
-  fill_xyzt(orientation_rate, ModelConstants.T_IDXS, *net_output_data['plan'][0,:,Plan.ORIENTATION_RATE].T)
+  fill_xyzt(modelV2.position, ModelConstants.T_IDXS, *net_output_data['plan'][0,:,Plan.POSITION].T, *net_output_data['plan_stds'][0,:,Plan.POSITION].T)
+  fill_xyzt(modelV2.velocity, ModelConstants.T_IDXS, *net_output_data['plan'][0,:,Plan.VELOCITY].T)
+  fill_xyzt(modelV2.acceleration, ModelConstants.T_IDXS, *net_output_data['plan'][0,:,Plan.ACCELERATION].T)
+  fill_xyzt(modelV2.orientation, ModelConstants.T_IDXS, *net_output_data['plan'][0,:,Plan.T_FROM_CURRENT_EULER].T)
+  fill_xyzt(modelV2.orientationRate, ModelConstants.T_IDXS, *net_output_data['plan'][0,:,Plan.ORIENTATION_RATE].T)
 
   # temporal pose
   temporal_pose = modelV2.temporalPose
@@ -103,12 +96,10 @@ def fill_model_msg(base_msg: capnp._DynamicStructBuilder, extended_msg: capnp._D
   temporal_pose.rotStd = net_output_data['plan_stds'][0,0,Plan.ORIENTATION_RATE].tolist()
 
   # poly path
-  poly_path = driving_model_data.path
-  fill_xyz_poly(poly_path, ModelConstants.POLY_PATH_DEGREE, *net_output_data['plan'][0,:,Plan.POSITION].T)
+  fill_xyz_poly(driving_model_data.path, ModelConstants.POLY_PATH_DEGREE, *net_output_data['plan'][0,:,Plan.POSITION].T)
 
   # lateral planning
-  action = modelV2.action
-  action.desiredCurvature = float(net_output_data['desired_curvature'][0,0])
+  modelV2.action.desiredCurvature = float(net_output_data['desired_curvature'][0,0])
 
   # times at X_IDXS according to model plan
   PLAN_T_IDXS = [np.nan] * ModelConstants.IDX_N
@@ -137,8 +128,7 @@ def fill_model_msg(base_msg: capnp._DynamicStructBuilder, extended_msg: capnp._D
   modelV2.laneLineStds = net_output_data['lane_lines_stds'][0,:,0,0].tolist()
   modelV2.laneLineProbs = net_output_data['lane_lines_prob'][0,1::2].tolist()
 
-  lane_line_meta = driving_model_data.laneLineMeta
-  fill_lane_line_meta(lane_line_meta, modelV2.laneLines, modelV2.laneLineProbs)
+  fill_lane_line_meta(driving_model_data.laneLineMeta, modelV2.laneLines, modelV2.laneLineProbs)
 
   # road edges
   modelV2.init('roadEdges', 2)

selfdrive/pandad/pandad.cc

@@ -91,9 +91,6 @@ void can_send_thread(std::vector<Panda *> pandas, bool fake_send) {
   while (!do_exit && check_all_connected(pandas)) {
     std::unique_ptr<Message> msg(subscriber->receive());
     if (!msg) {
-      if (errno == EINTR) {
-        do_exit = true;
-      }
       continue;
     }
 

selfdrive/ui/SConscript

@@ -67,7 +67,6 @@ if GetOption('extras'):
   qt_src.remove("main.cc")  # replaced by test_runner
   qt_env.Program('tests/test_translations', [asset_obj, 'tests/test_runner.cc', 'tests/test_translations.cc'] + qt_src, LIBS=qt_libs)
 
-if GetOption('extras'):
   qt_env.SharedLibrary("qt/python_helpers", ["qt/qt_window.cc"], LIBS=qt_libs)
 
   # spinner and text window

selfdrive/ui/installer/installer.cc

@@ -1,4 +1,3 @@
-#include <time.h>
 #include <unistd.h>
 
 #include <cstdlib>
@@ -11,6 +10,7 @@
 #include <QTimer>
 #include <QVBoxLayout>
 
+#include "common/util.h"
 #include "selfdrive/ui/installer/installer.h"
 #include "selfdrive/ui/qt/util.h"
 #include "selfdrive/ui/qt/qt_window.h"
@@ -36,14 +36,6 @@ const QString CACHE_PATH = "/data/openpilot.cache";
 extern const uint8_t str_continue[] asm("_binary_selfdrive_ui_installer_continue_openpilot_sh_start");
 extern const uint8_t str_continue_end[] asm("_binary_selfdrive_ui_installer_continue_openpilot_sh_end");
 
-bool time_valid() {
-  time_t rawtime;
-  time(&rawtime);
-
-  struct tm * sys_time = gmtime(&rawtime);
-  return (1900 + sys_time->tm_year) >= 2020;
-}
-
 void run(const char* cmd) {
   int err = std::system(cmd);
   assert(err == 0);
@@ -103,7 +95,7 @@ void Installer::updateProgress(int percent) {
 
 void Installer::doInstall() {
   // wait for valid time
-  while (!time_valid()) {
+  while (!util::system_time_valid()) {
     usleep(500 * 1000);
     qDebug() << "Waiting for valid time";
   }

selfdrive/ui/qt/offroad/developer_panel.cc

@@ -24,6 +24,13 @@ DeveloperPanel::DeveloperPanel(SettingsWindow *parent) : ListWidget(parent) {
   });
   addItem(longManeuverToggle);
 
+  alphaLongToggle = new ParamControl("ExperimentalLongitudinalEnabled", tr("openpilot Longitudinal Control (Alpha)"), "", "../assets/offroad/icon_speed_limit.png");
+  QObject::connect(alphaLongToggle, &ParamControl::toggleFlipped, [=](bool state) {
+    updateToggles(offroad);
+  });
+  addItem(alphaLongToggle);
+  alphaLongToggle->setConfirmation(true, false);
+
   // Joystick and longitudinal maneuvers should be hidden on release branches
   is_release = params.getBool("IsReleaseBranch");
 
@@ -43,8 +50,29 @@ void DeveloperPanel::updateToggles(bool _offroad) {
     AlignedBuffer aligned_buf;
     capnp::FlatArrayMessageReader cmsg(aligned_buf.align(cp_bytes.data(), cp_bytes.size()));
     cereal::CarParams::Reader CP = cmsg.getRoot<cereal::CarParams>();
+
+    const QString alpha_long_description = QString("<b>%1</b><br><br>%2")
+      .arg(tr("WARNING: openpilot longitudinal control is in alpha for this car and will disable Automatic Emergency Braking (AEB)."))
+      .arg(tr("On this car, openpilot defaults to the car's built-in ACC instead of openpilot's longitudinal control. "
+              "Enable this to switch to openpilot longitudinal control. Enabling Experimental mode is recommended when enabling openpilot longitudinal control alpha."));
+    alphaLongToggle->setDescription("<b>" + alpha_long_description + "</b>");
+
+    if (!CP.getExperimentalLongitudinalAvailable() && !CP.getOpenpilotLongitudinalControl()) {
+      params.remove("ExperimentalLongitudinalEnabled");
+      alphaLongToggle->setEnabled(false);
+      alphaLongToggle->setDescription("<b>" + tr("openpilot longitudinal control may come in a future update.") + "</b>");
+    }
+
+    // if is a release branch or if the car already have long control the alphaLongToggle should not be visible
+    if (is_release || CP.getOpenpilotLongitudinalControl()) {
+      params.remove("ExperimentalLongitudinalEnabled");
+      alphaLongToggle->setVisible(false);
+    }
+
+    alphaLongToggle->refresh();
     longManeuverToggle->setEnabled(hasLongitudinalControl(CP) && _offroad);
   } else {
+    alphaLongToggle->setVisible(false);
     longManeuverToggle->setEnabled(false);
   }
 

selfdrive/ui/qt/offroad/developer_panel.h

@@ -12,6 +12,8 @@ class DeveloperPanel : public ListWidget {
   Params params;
   ParamControl* joystickToggle;
   ParamControl* longManeuverToggle;
+  ParamControl* alphaLongToggle;
+
   bool is_release;
   bool offroad;