diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cb8cd30d..30ee697e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,8 +2,8 @@ Changelog ========= -[unreleased] -============ +[20230114] +========== ouster_client -------------- @@ -20,6 +20,11 @@ ouster_client * added a new method ``init_logger()`` to provide control over the logs emitted by ``ouster_client``. * add parsing for new FW 3.0 thermal features shot_limiting and thermal_shutdown statuses and countdowns * add frame_status to LidarScan +* introduce a new method ``cartesianT()`` which speeds up the computation of point projecion from range + image, the method also can process the cartesian product with single float precision. A new unit test + ``cartesian_test`` which shows achieved speed up gains by the number of valid returns in lidar scan. +* add ``RAW_HEADERS`` ChanField to LidarScan for packing headers and footer (alpha version, may be + changed/removed without notice in the future) python ------ @@ -27,6 +32,7 @@ python * breaking change: change Scans interface Timeout to default to 1 second instead of None * added a new method ``init_logger()`` to provide control over the logs emitted by ``client.Sensor``. * add ``client.LidarScan`` methods ``__repr__()`` and ``__str__()``. +* changed default timeout from 1 seconds to 2 seconds ouster_viz ---------- diff --git a/CMakeLists.txt b/CMakeLists.txt index 5eb3737f..93b9a310 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,10 +7,10 @@ include(DefaultBuildType) include(VcpkgEnv) # ==== Project Name ==== -project(ouster_example VERSION 20220927) +project(ouster_example VERSION 20230114) # generate version header -set(OusterSDK_VERSION_STRING 0.7.1.b1) +set(OusterSDK_VERSION_STRING 0.7.1) include(VersionGen) # ==== Options ==== diff --git a/conanfile.py b/conanfile.py index 008608e4..5e36fd81 100644 --- a/conanfile.py +++ b/conanfile.py @@ -82,7 +82,7 @@ def configure_cmake(self): cmake = CMake(self) cmake.definitions["BUILD_VIZ"] = self.options.build_viz cmake.definitions["BUILD_PCAP"] = self.options.build_pcap - cmake.definitions["USE_EIGEN_MAX_ALIGN_BYTES_32"] = self.options.eigen_max_align_bytes + cmake.definitions["OUSTER_USE_EIGEN_MAX_ALIGN_BYTES_32"] = self.options.eigen_max_align_bytes # alt way, but we use CMAKE_TOOLCHAIN_FILE in other pipeline so avoid overwrite # cmake.definitions["CMAKE_TOOLCHAIN_FILE"] = os.path.join(self.build_folder, "conan_paths.cmake") cmake.definitions[ diff --git a/docs/cpp/building.rst b/docs/cpp/building.rst index 4e9c3721..c548b970 100644 --- a/docs/cpp/building.rst +++ b/docs/cpp/building.rst @@ -28,7 +28,7 @@ On macOS, install XCode and `homebrew `_ and run: .. code:: console - $ brew install cmake pkg-config jsoncpp eigen curl libtins glfw glew + $ brew install cmake pkg-config jsoncpp eigen curl libtins glfw glew spdlog To build run the following commands: @@ -79,7 +79,7 @@ You should be able to install dependencies with .. code:: powershell - PS > .\vcpkg.exe install --triplet x64-windows jsoncpp eigen3 curl libtins glfw3 glew + PS > .\vcpkg.exe install --triplet x64-windows jsoncpp eigen3 curl libtins glfw3 glew spdlog After these steps are complete, you should be able to open, build and run the ``ouster_example`` project using Visual Studio: diff --git a/docs/installation.rst b/docs/installation.rst index 676e215c..f4745ad4 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -5,7 +5,7 @@ Ouster SDK Installation ======================= -Please proceed to the instructions for your language and platform of choice. Newer users may find the Python SDK more approachable. +Please proceed to the instructions for your programming language and platform of choice. .. contents:: :local: diff --git a/docs/python/devel.rst b/docs/python/devel.rst index baf0f168..f758c461 100644 --- a/docs/python/devel.rst +++ b/docs/python/devel.rst @@ -40,7 +40,7 @@ On macos >= 10.13, using homebrew, you should be able to run: .. code:: console - $ brew install cmake eigen curl jsoncpp libtins python3 pybind11 glfw glew + $ brew install cmake eigen curl jsoncpp libtins python3 pybind11 glfw glew spdlog After you have the system dependencies, you can build the SDK with: @@ -74,7 +74,7 @@ package manager and run: .. code:: powershell - PS > vcpkg install --triplet=x64-windows eigen3 jsoncpp libtins pybind11 glfw3 glad[gl-api-33] + PS > vcpkg install --triplet=x64-windows eigen3 jsoncpp libtins pybind11 glfw3 glad[gl-api-33] spdlog The currently tested vcpkg tag is ``2022.02.23``. After that, using a developer powershell prompt: diff --git a/docs/reference/lidar-scan.rst b/docs/reference/lidar-scan.rst index 22e7296c..d7e14da4 100644 --- a/docs/reference/lidar-scan.rst +++ b/docs/reference/lidar-scan.rst @@ -231,4 +231,4 @@ both sampling, used in :ref:`ex-visualization-with-matplotlib`, and streaming, u Under the hood, this class batches packets into ``LidarScans``. C++ users must batch packets themselves using the :cpp:class:`ouster::ScanBatcher` class. To get a feel for how to use it, we recommend reading `this example on Github -`_. +`_. diff --git a/docs/sample-data.rst b/docs/sample-data.rst index 709548d2..52ffc52e 100644 --- a/docs/sample-data.rst +++ b/docs/sample-data.rst @@ -13,33 +13,16 @@ Download Data .. [start-download-instructions] -Download one of the following samples of recorded Ouster data and unzip the contents: - -.. _dual-returns-snippets: - - * `OS0 128 Rev 06 Urban Drive (Dual Returns)`_ [151 MB] (`preview OS0 `_) - * `OS1 128 Rev 06 Urban Drive (Dual Returns)`_ [173 MB] (`preview OS1 `_) - * `OS2 128 Rev 06 Urban Drive (Dual Returns)`_ [190 MB] (`preview OS2 `_) - * `OS1 128 Rev 06 Urban Drive (Low Bandwidth)`_ [95 MB] (`preview OS1 LB `_) - * `OS2 128 Rev 05 Bridge`_ [82 MB] (`preview bridge `_) - - -.. _OS0 128 Rev 06 Urban Drive (Dual Returns): https://data.ouster.io/sdk-samples/Rev-06-fw23/OS0-128_Rev-06_fw23_Urban-Drive_Dual-Returns.zip -.. _OS1 128 Rev 06 Urban Drive (Dual Returns): https://data.ouster.io/sdk-samples/Rev-06-fw23/OS1-128_Rev-06_fw23_Urban-Drive_Dual-Returns.zip -.. _OS2 128 Rev 06 Urban Drive (Dual Returns): https://data.ouster.io/sdk-samples/Rev-06-fw23/OS2-128_Rev-06_fw23_Urban-Drive_Dual-Returns.zip -.. _OS1 128 Rev 06 Urban Drive (Low Bandwidth): https://data.ouster.io/sdk-samples/Rev-06-fw23/OS1-128_Rev-06_fw23_Urban-Drive_Low-Bandwidth.zip -.. _OS2 128 Rev 05 Bridge: https://data.ouster.io/sdk-samples/Rev-05/OS2-128_Rev-05_Bridge/OS2-128_Rev-05_Bridge.zip - -In your unzipped directory, you should have two files, one ``.pcap`` file and one ``.json`` file. -For example, in the unzipped recorded sample of OS1-128 Rev 06 data you should find: - - * ``OS1-128_Rev-06_fw23_Urban-Drive_Dual-Returns.pcap`` - * ``OS1-128_Rev-06_fw23_Urban-Drive_Dual-Returns.json`` +You can download sample data from the `sensor documentation`_ by clicking through the Ouster Data +App links and using the Download button. After download, you should have two files, a ``.pcap`` file +and a ``.json`` file. We will use ``SAMPLE_DATA_PCAP_PATH`` to refer to this pcap and ``SAMPLE_DATA_JSON_PATH`` to this json in the following. You may find it convenient to assign the paths appropriately in your console. +.. _sensor documentation: https://static.ouster.dev/sensor-docs/#sample-data + .. [end-download-instructions] diff --git a/examples/representations_example.cpp b/examples/representations_example.cpp index d35813ae..f4189153 100644 --- a/examples/representations_example.cpp +++ b/examples/representations_example.cpp @@ -154,10 +154,9 @@ int main(int argc, char* argv[]) { std::to_string(print_column) + ")"; std::cerr << "In the staggered image, the point at " << point_string - << " has reflectivity " << reflectivity(print_row, print_row) + << " has reflectivity " << reflectivity(print_row, print_column) << " and an x coordinate of " - << x_image_staggered(print_column, print_column) << "." - << std::endl; + << x_image_staggered(print_row, print_column) << "." << std::endl; std::cerr << "In the destagged image, the point at " << point_string << " has reflectivity " << reflectivity_destaggered(print_row, print_column) diff --git a/ouster_client/include/ouster/impl/cartesian.h b/ouster_client/include/ouster/impl/cartesian.h new file mode 100644 index 00000000..24b73982 --- /dev/null +++ b/ouster_client/include/ouster/impl/cartesian.h @@ -0,0 +1,74 @@ +#pragma once + +#include + +namespace ouster { + +// Users can enable OpenMP within ouster_client by first enabling omp through +// the compiler and then adding '-DOUSTER_OMP' option to the DCMAKE_CXX_FLAGS +#if defined(OUSTER_OMP) +#if defined(_OPENMP) +#define __OUSTER_UTILIZE_OPENMP__ +#else +#pragma message("OUSTER_OMP was defined but openmp is not detected!") +#endif +#endif + +template +using PointsT = Eigen::Array; +using PointsD = PointsT; +using PointsF = PointsT; + +/** + * Converts a staggered range image to Cartesian points. + * + * @param[in, out] points The resulting point cloud, should be pre-allocated and + * have the same dimensions as the direction array. + * @param[in] range a range image in the same format as the RANGE field of a + * LidarScan. + * @param[in] direction the direction of an xyz lut. + * @param[in] offset the offset of an xyz lut. + * + * @return Cartesian points where ith row is a 3D point which corresponds + * to ith pixel in LidarScan where i = row * w + col. + */ +template +void cartesianT(PointsT& points, + const Eigen::Ref>& range, + const PointsT& direction, const PointsT& offset) { + assert(points.rows() == direction.rows() && + "points & direction row count mismatch"); + assert(points.rows() == offset.rows() && + "points & offset row count mismatch"); + assert(points.rows() == range.size() && + "points and range image size mismatch"); + + const auto pts = points.data(); + const auto* const rng = range.data(); + const auto* const dir = direction.data(); + const auto* const ofs = offset.data(); + + const auto N = range.size(); + const auto col_x = 0 * N; // 1st column of points (x) + const auto col_y = 1 * N; // 2nd column of points (y) + const auto col_z = 2 * N; // 3rd column of points (z) + +#ifdef __OUSTER_UTILIZE_OPENMP__ +#pragma omp parallel for schedule(static) +#endif + for (auto i = 0; i < N; ++i) { + const auto r = rng[i]; + const auto idx_x = col_x + i; + const auto idx_y = col_y + i; + const auto idx_z = col_z + i; + if (r == 0) { + pts[idx_x] = pts[idx_y] = pts[idx_z] = static_cast(0.0); + } else { + pts[idx_x] = r * dir[idx_x] + ofs[idx_x]; + pts[idx_y] = r * dir[idx_y] + ofs[idx_y]; + pts[idx_z] = r * dir[idx_z] + ofs[idx_z]; + } + } +} + +} // namespace ouster diff --git a/ouster_client/include/ouster/lidar_scan.h b/ouster_client/include/ouster/lidar_scan.h index 9e655370..684b62ac 100644 --- a/ouster_client/include/ouster/lidar_scan.h +++ b/ouster_client/include/ouster/lidar_scan.h @@ -519,7 +519,8 @@ inline img_t stagger(const Eigen::Ref>& img, class ScanBatcher { std::ptrdiff_t w; std::ptrdiff_t h; - uint16_t next_m_id; + uint16_t next_valid_m_id; + uint16_t next_headers_m_id; std::vector cache; bool cached_packet = false; @@ -555,4 +556,5 @@ class ScanBatcher { } // namespace ouster +#include "ouster/impl/cartesian.h" #include "ouster/impl/lidar_scan_impl.h" diff --git a/ouster_client/include/ouster/types.h b/ouster_client/include/ouster/types.h index 4a9344a3..eaa6f49b 100644 --- a/ouster_client/include/ouster/types.h +++ b/ouster_client/include/ouster/types.h @@ -655,6 +655,13 @@ std::string to_string(ShotLimitingStatus shot_limiting_status); */ std::string to_string(ThermalShutdownStatus thermal_shutdown_status); +/** + * Determine validity of provided signal multiplier value + * + * @param[in] signal_multiplier Signal multiplier value. + */ +void check_signal_multiplier(const double signal_multiplier); + /** * Parse metadata text blob from the sensor into a sensor_info struct. * @@ -747,6 +754,7 @@ enum ChanField { NEAR_IR = 7, ///< near_ir in photons FLAGS = 8, ///< 1st return flags FLAGS2 = 9, ///< 2nd return flags + RAW_HEADERS = 40, ///< raw headers for packet/footer/column for dev use CUSTOM0 = 50, ///< custom user field CUSTOM1 = 51, ///< custom user field CUSTOM2 = 52, ///< custom user field @@ -779,6 +787,15 @@ std::string to_string(ChanField field); */ enum ChanFieldType { VOID = 0, UINT8, UINT16, UINT32, UINT64 }; +/** + * Get the size of the ChanFieldType in bytes. + * + * @param[in] ft the field type + * + * @return size of the field type in bytes + */ +size_t field_type_size(ChanFieldType ft); + /** * Get string representation of a channel field. * @@ -827,6 +844,12 @@ class packet_format final { const int pixels_per_column; ///< pixels per column for lidar [[deprecated]] const int encoder_ticks_per_rev; ///< @deprecated + const size_t packet_header_size; + const size_t col_header_size; + const size_t col_footer_size; + const size_t col_size; + const size_t packet_footer_size; + /** * Read the packet type packet header. * @@ -919,6 +942,16 @@ class packet_format final { */ FieldIter end() const; + /** + * Get pointer to the packet footer of a lidar buffer. + * + * @param[in] lidar_buf the lidar buffer. + * + * @return pointer to packet footer of lidar buffer, can be nullptr if + * packet format doesn't have packet footer. + */ + const uint8_t* footer(const uint8_t* lidar_buf) const; + // Measurement block accessors /** * Get pointer to nth column of a lidar buffer. diff --git a/ouster_client/src/client.cpp b/ouster_client/src/client.cpp index 6c35c62a..c687adb5 100644 --- a/ouster_client/src/client.cpp +++ b/ouster_client/src/client.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -215,14 +216,13 @@ bool set_config(const std::string& hostname, const sensor_config& config, // Signal multiplier changed from int to double for FW 3.0/2.5+, with // corresponding change to config.signal_multiplier. // Change values 1, 2, 3 back to ints to support older FWs - if (config_params["signal_multiplier"].asDouble() != 0.25 && - config_params["signal_multiplier"].asDouble() != 0.5) { - config_params["signal_multiplier"] = - config_params["signal_multiplier"].asInt(); - // TODO: figure out what to do when people enter invalid signal - // multiplier values between 1 and 3.99. Invalid decimal values <1 and - // >4 will be invalid in any case and the error message won't say so - // those are fine + if (config_json.isMember("signal_multiplier")) { + check_signal_multiplier(config_params["signal_multiplier"].asDouble()); + if (config_params["signal_multiplier"].asDouble() != 0.25 && + config_params["signal_multiplier"].asDouble() != 0.5) { + config_params["signal_multiplier"] = + config_params["signal_multiplier"].asInt(); + } } // set automatic udp dest, if flag specified @@ -279,6 +279,34 @@ std::string get_metadata(client& cli, int timeout_sec, bool legacy_format) { builder["precision"] = 6; builder["indentation"] = " "; auto metadata_string = Json::writeString(builder, cli.meta); + if (legacy_format) { + logger().warn( + "The SDK will soon output the non-legacy metadata format by " + "default. If you parse the metadata directly instead of using the " + "SDK (which will continue to read both legacy and non-legacy " + "formats), please be advised that on the next release you will " + "either have to update your parsing or specify legacy_format = " + "true to the get_metadata function."); + } + + // We can't insert this logic into the light init_client since its advantage + // is that it doesn't make netowrk calls but we need it to run every time + // there is a valid connection to the sensor So we insert it here + // TODO: remove after release of FW 3.2/3.3 (sufficient warning) + sensor_config config; + get_config(cli.hostname, config); + auto fw_version = SensorHttp::firmware_version(cli.hostname); + // only warn for people on the latest FW, as people on older FWs may not + // care + if (fw_version.major >= 3 && + config.udp_profile_lidar == UDPProfileLidar::PROFILE_LIDAR_LEGACY) { + logger().warn( + "Please note that the Legacy Lidar Profile will be deprecated " + "in the sensor FW soon. If you plan to upgrade your FW, we " + "recommend using the Single Return Profile instead. For users " + "sticking with older FWs, the Ouster SDK will continue to parse " + "the legacy lidar profile."); + } return legacy_format ? convert_to_legacy(metadata_string) : metadata_string; } diff --git a/ouster_client/src/curl_client.h b/ouster_client/src/curl_client.h index eea30c97..2b93afc8 100644 --- a/ouster_client/src/curl_client.h +++ b/ouster_client/src/curl_client.h @@ -1,7 +1,6 @@ #include #include -#include #include "http_client.h" diff --git a/ouster_client/src/lidar_scan.cpp b/ouster_client/src/lidar_scan.cpp index e9ff336d..cfb3ce92 100644 --- a/ouster_client/src/lidar_scan.cpp +++ b/ouster_client/src/lidar_scan.cpp @@ -9,10 +9,10 @@ #include #include #include -#include #include #include +#include "logging.h" #include "ouster/impl/lidar_scan_impl.h" #include "ouster/types.h" @@ -384,7 +384,6 @@ LidarScan::Points cartesian(const Eigen::Ref>& range, const XYZLut& lut) { if (range.cols() * range.rows() != lut.direction.rows()) throw std::invalid_argument("unexpected image dimensions"); - auto reshaped = Eigen::Map>( range.data(), range.cols() * range.rows()); auto nooffset = lut.direction.colwise() * reshaped.cast(); @@ -394,7 +393,8 @@ LidarScan::Points cartesian(const Eigen::Ref>& range, ScanBatcher::ScanBatcher(size_t w, const sensor::packet_format& pf) : w(w), h(pf.pixels_per_column), - next_m_id(0), + next_valid_m_id(0), + next_headers_m_id(0), cache(pf.lidar_packet_size), pf(pf) {} @@ -434,7 +434,14 @@ struct parse_field_col { template void operator()(Eigen::Ref> field, ChanField f, uint16_t m_id, const sensor::packet_format& pf, const uint8_t* col_buf) { + // user defined fields that we shouldn't change if (f >= ChanField::CUSTOM0 && f <= ChanField::CUSTOM9) return; + + // RAW_HEADERS field is populated separately because it has + // a different processing scheme and doesn't fit into existing field + // model (i.e. data packed per column rather than per pixel) + if (f == ChanField::RAW_HEADERS) return; + pf.col_field(col_buf, f, field.col(m_id).data(), field.cols()); } }; @@ -442,16 +449,90 @@ struct parse_field_col { uint64_t frame_status(const uint8_t thermal_shutdown, const uint8_t shot_limiting) { uint64_t res = 0; + + // clang-format off res |= (thermal_shutdown & 0x0f) - << FRAME_STATUS_THERMAL_SHUTDOWN_SHIFT; // right nibble is thermal - // shutdown status, apply mask for - // safety, then shift + << FRAME_STATUS_THERMAL_SHUTDOWN_SHIFT; // right nibble is thermal + // shutdown status, apply mask + // for safety, then shift + //clang-format on res |= (shot_limiting & 0x0f) - << FRAME_STATUS_SHOT_LIMITING_SHIFT; // right nibble is shot limiting, apply mask for + << FRAME_STATUS_SHOT_LIMITING_SHIFT; // right nibble is shot + // limiting, apply mask for // safety, then shift return res; } +/** + * Checks whether RAW_HEADERS field is present and can be used to store headers. + * + * @param[in] pf packet format + * @param[in] ls lidar scan to check for RAW_HEADERS field presence. + */ +bool raw_headers_enabled(const sensor::packet_format& pf, const LidarScan& ls) { + using ouster::sensor::logger; + ChanFieldType raw_headers_ft = ls.field_type(ChanField::RAW_HEADERS); + if (!raw_headers_ft) { + return false; + } + // ensure that we can pack headers into the size of a single RAW_HEADERS + // column + if (pf.pixels_per_column * sensor::field_type_size(raw_headers_ft) < + (pf.packet_header_size + pf.col_header_size + pf.col_footer_size + + pf.packet_footer_size)) { + logger().debug( + "WARNING: Can't fit RAW_HEADERS into a column of {} {} " + "values", + pf.pixels_per_column, to_string(raw_headers_ft)); + return false; + } + return true; +} + +/** + * Pack the lidar packet and column headers and footer into a RAW_HEADERS field. + */ +struct pack_raw_headers_col { + template + void operator()(Eigen::Ref> rh_field, ChanField, + const sensor::packet_format& pf, uint16_t col_idx, + const uint8_t* packet_buf) { + const uint8_t* col_buf = pf.nth_col(col_idx, packet_buf); + const uint16_t m_id = pf.col_measurement_id(col_buf); + + using ColMajorView = + Eigen::Map>; + + const ColMajorView col_header_vec(reinterpret_cast(col_buf), + pf.col_header_size / sizeof(T)); + + rh_field.block(0, m_id, col_header_vec.size(), 1) = col_header_vec; + + const ColMajorView col_footer_vec( + reinterpret_cast(col_buf + pf.col_size - + pf.col_footer_size), + pf.col_footer_size / sizeof(T)); + + rh_field.block(col_header_vec.size(), m_id, col_footer_vec.size(), 1) = + col_footer_vec; + + const ColMajorView packet_header_vec( + reinterpret_cast(packet_buf), + pf.packet_header_size / sizeof(T)); + + rh_field.block(col_header_vec.size() + col_footer_vec.size(), m_id, + packet_header_vec.size(), 1) = packet_header_vec; + + const ColMajorView packet_footer_vec( + reinterpret_cast(pf.footer(packet_buf)), + pf.packet_footer_size / sizeof(T)); + + rh_field.block(col_header_vec.size() + col_footer_vec.size() + + packet_header_vec.size(), + m_id, packet_footer_vec.size(), 1) = packet_footer_vec; + } +}; + } // namespace bool ScanBatcher::operator()(const uint8_t* packet_buf, LidarScan& ls) { @@ -467,9 +548,12 @@ bool ScanBatcher::operator()(const uint8_t* packet_buf, LidarScan& ls) { const uint16_t f_id = pf.frame_id(packet_buf); + const bool raw_headers = raw_headers_enabled(pf, ls); + if (ls.frame_id == -1) { // expecting to start batching a new scan - next_m_id = 0; + next_valid_m_id = 0; + next_headers_m_id = 0; ls.frame_id = f_id; const uint8_t f_thermal_shutdown = pf.thermal_shutdown(packet_buf); @@ -481,8 +565,16 @@ bool ScanBatcher::operator()(const uint8_t* packet_buf, LidarScan& ls) { return false; } else if (ls.frame_id != f_id) { // got a packet from a new frame - impl::foreach_field(ls, zero_field_cols(), next_m_id, w); - zero_header_cols(ls, next_m_id, w); + for (const auto& field_type : ls) { + auto end_m_id = next_valid_m_id; + if (raw_headers && field_type.first == ChanField::RAW_HEADERS) { + end_m_id = next_headers_m_id; + } + impl::visit_field(ls, field_type.first, zero_field_cols(), + field_type.first, end_m_id, w); + } + + zero_header_cols(ls, next_valid_m_id, w); std::memcpy(cache.data(), packet_buf, cache.size()); cached_packet = true; @@ -498,14 +590,35 @@ bool ScanBatcher::operator()(const uint8_t* packet_buf, LidarScan& ls) { const uint32_t status = pf.col_status(col_buf); const bool valid = (status & 0x01); - // drop invalid / out-of-bounds data in case of misconfiguration - if (!valid || m_id >= w) continue; + // drop out-of-bounds data in case of misconfiguration + if (m_id >= w) continue; + + if (raw_headers) { + // zero out missing columns if we jumped forward + if (m_id >= next_headers_m_id) { + impl::visit_field(ls, ChanField::RAW_HEADERS, zero_field_cols(), + ChanField::RAW_HEADERS, next_headers_m_id, + m_id); + next_headers_m_id = m_id + 1; + } + + impl::visit_field(ls, ChanField::RAW_HEADERS, + pack_raw_headers_col(), ChanField::RAW_HEADERS, + pf, icol, packet_buf); + } + + // drop invalid + if (!valid) continue; // zero out missing columns if we jumped forward - if (m_id >= next_m_id) { - impl::foreach_field(ls, zero_field_cols(), next_m_id, m_id); - zero_header_cols(ls, next_m_id, m_id); - next_m_id = m_id + 1; + if (m_id >= next_valid_m_id) { + for (const auto& field_type : ls) { + if (field_type.first == ChanField::RAW_HEADERS) continue; + impl::visit_field(ls, field_type.first, zero_field_cols(), + field_type.first, next_valid_m_id, m_id); + } + zero_header_cols(ls, next_valid_m_id, m_id); + next_valid_m_id = m_id + 1; } // old header API; will be removed in a future release @@ -518,6 +631,7 @@ bool ScanBatcher::operator()(const uint8_t* packet_buf, LidarScan& ls) { impl::foreach_field(ls, parse_field_col(), m_id, pf, col_buf); } + return false; } // namespace ouster diff --git a/ouster_client/src/logging.cpp b/ouster_client/src/logging.cpp index 5ab4299e..e62f6cb4 100644 --- a/ouster_client/src/logging.cpp +++ b/ouster_client/src/logging.cpp @@ -100,4 +100,4 @@ namespace ouster { namespace sensor { spdlog::logger& logger() { return Logger::instance().get_logger(); } } // namespace sensor -} // namespace ouster \ No newline at end of file +} // namespace ouster diff --git a/ouster_client/src/parsing.cpp b/ouster_client/src/parsing.cpp index 8aed25d0..f3aaf73e 100644 --- a/ouster_client/src/parsing.cpp +++ b/ouster_client/src/parsing.cpp @@ -24,21 +24,6 @@ namespace impl { constexpr int imu_packet_size = 48; constexpr int64_t encoder_ticks_per_rev = 90112; -static size_t field_ty_size(ChanFieldType t) { - switch (t) { - case UINT8: - return 1; - case UINT16: - return 2; - case UINT32: - return 4; - case UINT64: - return 8; - default: - return 0; - } -} - template using Table = std::array, N>; @@ -176,7 +161,12 @@ packet_format::packet_format(const sensor_info& info) imu_packet_size{impl::imu_packet_size}, columns_per_packet(info.format.columns_per_packet), pixels_per_column(info.format.pixels_per_column), - encoder_ticks_per_rev{impl::encoder_ticks_per_rev} { + encoder_ticks_per_rev{impl::encoder_ticks_per_rev}, + packet_header_size{impl_->packet_header_size}, + col_header_size{impl_->col_header_size}, + col_footer_size{impl_->col_footer_size}, + col_size{impl_->col_size}, + packet_footer_size{impl_->packet_footer_size} { for (const auto& kv : impl_->fields) { field_types_.push_back({kv.first, kv.second.ty_tag}); } @@ -345,6 +335,12 @@ uint8_t packet_format::shot_limiting(const uint8_t* lidar_buf) const { return res & 0x0f; } +const uint8_t* packet_format::footer(const uint8_t* lidar_buf) const { + if (impl_->packet_footer_size == 0) return nullptr; + return lidar_buf + impl_->packet_header_size + + (columns_per_packet * impl_->col_size); +} + /* Measurement block access */ const uint8_t* packet_format::nth_col(int n, const uint8_t* lidar_buf) const { @@ -403,11 +399,11 @@ template T packet_format::px_field(const uint8_t* px_buf, ChanField i) const { const auto& f = impl_->fields.at(i); - if (sizeof(T) < impl::field_ty_size(f.ty_tag)) + if (sizeof(T) < field_type_size(f.ty_tag)) throw std::invalid_argument("Dest type too small for specified field"); T res = 0; - std::memcpy(&res, px_buf + f.offset, impl::field_ty_size(f.ty_tag)); + std::memcpy(&res, px_buf + f.offset, field_type_size(f.ty_tag)); if (f.mask) res &= f.mask; if (f.shift > 0) res >>= f.shift; if (f.shift < 0) res <<= std::abs(f.shift); diff --git a/ouster_client/src/types.cpp b/ouster_client/src/types.cpp index 7bcf707a..824911c8 100644 --- a/ouster_client/src/types.cpp +++ b/ouster_client/src/types.cpp @@ -20,7 +20,6 @@ #include "logging.h" #include "ouster/impl/build.h" #include "ouster/version.h" - namespace ouster { using nonstd::make_optional; @@ -69,7 +68,7 @@ extern const Table polarity_strings{ extern const Table nmea_baud_rate_strings{ {{BAUD_9600, "BAUD_9600"}, {BAUD_115200, "BAUD_115200"}}}; -Table chanfield_strings{{ +Table chanfield_strings{{ {ChanField::RANGE, "RANGE"}, {ChanField::RANGE2, "RANGE2"}, {ChanField::SIGNAL, "SIGNAL"}, @@ -79,6 +78,7 @@ Table chanfield_strings{{ {ChanField::NEAR_IR, "NEAR_IR"}, {ChanField::FLAGS, "FLAGS"}, {ChanField::FLAGS2, "FLAGS2"}, + {ChanField::RAW_HEADERS, "RAW_HEADERS"}, {ChanField::CUSTOM0, "CUSTOM0"}, {ChanField::CUSTOM1, "CUSTOM1"}, {ChanField::CUSTOM2, "CUSTOM2"}, @@ -451,6 +451,21 @@ std::string to_string(ChanFieldType ft) { } } +size_t field_type_size(ChanFieldType ft) { + switch (ft) { + case sensor::ChanFieldType::UINT8: + return 1; + case sensor::ChanFieldType::UINT16: + return 2; + case sensor::ChanFieldType::UINT32: + return 4; + case sensor::ChanFieldType::UINT64: + return 8; + default: + return 0; + } +} + std::string to_string(UDPProfileLidar profile) { auto res = lookup(impl::udp_profile_lidar_strings, profile); return res ? res.value() : "UNKNOWN"; @@ -480,6 +495,28 @@ std::string to_string(ThermalShutdownStatus thermal_shutdown_status) { return res ? res.value() : "UNKNOWN"; } +void check_signal_multiplier(const double signal_multiplier) { + int signal_multiplier_int = int(signal_multiplier); + std::string signal_multiplier_error = + "Provided signal multiplier is invalid: " + + std::to_string(signal_multiplier) + + " cannot be converted to one of [0.25, 0.5, 1, 2, 3]"; + + // get the doubles out of the way + if (signal_multiplier == 0.25 || signal_multiplier == 0.5) return; + + // everything else has to be essentially an int + if (std::fabs(signal_multiplier - double(signal_multiplier_int)) > + signal_multiplier_int * std::numeric_limits::epsilon()) { + throw std::runtime_error(signal_multiplier_error); + } + + // the int has to 1, 2, or 3 + if (signal_multiplier_int < 1 || signal_multiplier_int > 3) { + throw std::runtime_error(signal_multiplier_error); + } +} + static sensor_config parse_config(const Json::Value& root) { sensor_config config{}; @@ -508,8 +545,11 @@ static sensor_config parse_config(const Json::Value& root) { std::make_pair(root["azimuth_window"][0].asInt(), root["azimuth_window"][1].asInt()); - if (!root["signal_multiplier"].empty()) - config.signal_multiplier = root["signal_multiplier"].asDouble(); + if (!root["signal_multiplier"].empty()) { + double signal_multiplier = root["signal_multiplier"].asDouble(); + check_signal_multiplier(signal_multiplier); + config.signal_multiplier = signal_multiplier; + } if (!root["operating_mode"].empty()) { auto operating_mode = @@ -517,7 +557,7 @@ static sensor_config parse_config(const Json::Value& root) { if (operating_mode) { config.operating_mode = operating_mode; } else { - throw std::invalid_argument("Unexpected Operating Mode"); + throw std::runtime_error{"Unexpected Operating Mode"}; } } else if (!root["auto_start_flag"].empty()) { logger().warn( @@ -534,7 +574,7 @@ static sensor_config parse_config(const Json::Value& root) { if (multipurpose_io_mode) { config.multipurpose_io_mode = multipurpose_io_mode; } else { - throw std::invalid_argument("Unexpected Multipurpose IO Mode"); + throw std::runtime_error{"Unexpected Multipurpose IO Mode"}; } } if (!root["sync_pulse_out_angle"].empty()) @@ -549,7 +589,7 @@ static sensor_config parse_config(const Json::Value& root) { if (nmea_in_polarity) { config.nmea_in_polarity = nmea_in_polarity; } else { - throw std::invalid_argument("Unexpected NMEA Input Polarity"); + throw std::runtime_error{"Unexpected NMEA Input Polarity"}; } } if (!root["nmea_baud_rate"].empty()) { @@ -558,7 +598,7 @@ static sensor_config parse_config(const Json::Value& root) { if (nmea_baud_rate) { config.nmea_baud_rate = nmea_baud_rate; } else { - throw std::invalid_argument("Unexpected NMEA Baud Rate"); + throw std::runtime_error{"Unexpected NMEA Baud Rate"}; } } if (!root["nmea_ignore_valid_char"].empty()) @@ -572,7 +612,7 @@ static sensor_config parse_config(const Json::Value& root) { if (sync_pulse_in_polarity) { config.sync_pulse_in_polarity = sync_pulse_in_polarity; } else { - throw std::invalid_argument("Unexpected Sync Pulse Input Polarity"); + throw std::runtime_error{"Unexpected Sync Pulse Input Polarity"}; } } if (!root["sync_pulse_out_polarity"].empty()) { @@ -581,8 +621,7 @@ static sensor_config parse_config(const Json::Value& root) { if (sync_pulse_out_polarity) { config.sync_pulse_out_polarity = sync_pulse_out_polarity; } else { - throw std::invalid_argument( - "Unexpected Sync Pulse Output Polarity"); + throw std::runtime_error{"Unexpected Sync Pulse Output Polarity"}; } } if (!root["sync_pulse_out_frequency"].empty()) @@ -619,20 +658,24 @@ sensor_config parse_config(const std::string& config) { if (config.size()) { if (!Json::parseFromStream(builder, ss, &root, &errors)) { - throw std::invalid_argument{errors.c_str()}; + throw std::runtime_error{errors}; } } return parse_config(root); } -static bool valid_response(const Json::Value& root, - const std::string& tcp_request) { - return (root.isMember(tcp_request) && root[tcp_request].isObject()); -} +// bool represents whether it is an object (true) or just a member (false) +// NOTE: lidar_data_format and calibration_status should be objects but as they +// were introduced earlier, non-legacy formats for FW version do not include +// them +// TODO parse metadata by FW version specified ? +const std::map nonlegacy_metadata_fields = { + {"sensor_info", true}, {"beam_intrinsics", true}, + {"imu_intrinsics", true}, {"lidar_intrinsics", true}, + {"config_params", true}, {"lidar_data_format", false}, + {"calibration_status", false}}; -// TODO make robust to new formats that are incorrect instead of returning -// false and sending to legacy static bool is_new_format(const std::string& metadata) { Json::Value root{}; Json::CharReaderBuilder builder{}; @@ -641,32 +684,45 @@ static bool is_new_format(const std::string& metadata) { if (metadata.size()) { if (!Json::parseFromStream(builder, ss, &root, &errors)) - throw std::invalid_argument{errors.c_str()}; + throw std::runtime_error{errors}; } - const std::vector valid_response_required = { - "sensor_info", "beam_intrinsics", "imu_intrinsics", "lidar_intrinsics", - "config_params"}; - - for (const auto& key : valid_response_required) { - if (!valid_response(root, key)) { - return false; + size_t nonlegacy_fields_present = 0; + std::string missing_fields = ""; + for (const auto& field_pair : nonlegacy_metadata_fields) { + auto field = field_pair.first; + auto is_obj = field_pair.second; + if (root.isMember(field)) { + nonlegacy_fields_present++; + if (is_obj && !root[field].isObject()) { + throw std::runtime_error{"Non-legacy metadata field " + field + + " must have child fields"}; + } + } else { + missing_fields += field + " "; } } - const std::vector member_required = {"lidar_data_format", - "calibration_status"}; - - for (const auto& key : member_required) { - if (!root.isMember(key)) { - return false; - } + if (nonlegacy_fields_present > 0 && + nonlegacy_fields_present < nonlegacy_metadata_fields.size()) { + throw std::runtime_error{"Non-legacy metadata must include fields: " + + missing_fields}; } - return true; + return nonlegacy_fields_present == nonlegacy_metadata_fields.size(); } static data_format parse_data_format(const Json::Value& root) { + const std::vector data_format_required_fields{ + "pixels_per_column", "columns_per_packet", "columns_per_frame", + "pixel_shift_by_row"}; + + for (const auto& field : data_format_required_fields) { + if (!root.isMember(field)) { + throw std::runtime_error{ + "Metadata field data_format must include field: " + field}; + } + } data_format format; format.pixels_per_column = root["pixels_per_column"].asInt(); @@ -674,7 +730,7 @@ static data_format parse_data_format(const Json::Value& root) { format.columns_per_frame = root["columns_per_frame"].asInt(); if (root["pixel_shift_by_row"].size() != format.pixels_per_column) { - throw std::invalid_argument{"Unexpected number of pixel_shift_by_row"}; + throw std::runtime_error{"Unexpected number of pixel_shift_by_row"}; } for (const auto& v : root["pixel_shift_by_row"]) @@ -682,28 +738,29 @@ static data_format parse_data_format(const Json::Value& root) { if (root.isMember("column_window")) { if (root["column_window"].size() != 2) { - throw std::invalid_argument{ - "Unexpected size of column_window tuple"}; + throw std::runtime_error{"Unexpected size of column_window tuple"}; } format.column_window.first = root["column_window"][0].asInt(); format.column_window.second = root["column_window"][1].asInt(); } else { - logger().warn("No column window found."); + logger().warn( + "No column window found. Using default column window (full)"); format.column_window = default_column_window(format.columns_per_frame); } if (root.isMember("udp_profile_lidar")) { - // initializing directly triggers -Wmaybe-uninitialized GCC 8.3.1 + // initializing directly triggers -Wmaybe-uninitialized + // GCC 8.3.1 optional profile{nullopt}; profile = udp_profile_lidar_of_string(root["udp_profile_lidar"].asString()); if (profile) { format.udp_profile_lidar = profile.value(); } else { - throw std::invalid_argument{"Unexpected udp lidar profile"}; + throw std::runtime_error{"Unexpected udp lidar profile"}; } } else { - logger().warn("No lidar profile found."); + logger().warn("No lidar profile found. Using LEGACY lidar profile"); format.udp_profile_lidar = PROFILE_LIDAR_LEGACY; } @@ -713,14 +770,15 @@ static data_format parse_data_format(const Json::Value& root) { if (profile) { format.udp_profile_imu = profile.value(); } else { - throw std::invalid_argument{"Unexpected udp imu profile"}; + throw std::runtime_error{"Unexpected udp imu profile"}; } } else { + logger().warn("No imu profile found. Using LEGACY imu profile"); format.udp_profile_imu = PROFILE_IMU_LEGACY; } return format; -} +} // namespace sensor static sensor_info parse_legacy(const std::string& meta) { Json::Value root{}; @@ -730,29 +788,71 @@ static sensor_info parse_legacy(const std::string& meta) { if (meta.size()) { if (!Json::parseFromStream(builder, ss, &root, &errors)) - throw std::invalid_argument{errors.c_str()}; + throw std::runtime_error{errors}; + } + + const std::vector minimum_legacy_metadata_fields{ + "beam_altitude_angles", "beam_azimuth_angles", "lidar_mode"}; + + for (auto field : minimum_legacy_metadata_fields) { + if (!root.isMember(field)) { + throw std::runtime_error{"Metadata must contain: " + field}; + } + } + + // nice to have fields which we will use defaults for if they don't + // exist + const std::vector desired_legacy_metadata_fields{ + "imu_to_sensor_transform", "lidar_to_sensor_transform", "prod_line", + "prod_sn", "build_rev"}; + for (auto field : desired_legacy_metadata_fields) { + if (!root.isMember(field)) { + logger().warn("No " + field + + " found in metadata. Will be left blank or filled in " + "with default legacy values"); + } + } + + // fields that don't survive round trip through to_string + const std::vector not_parsed_metadata_fields{ + "build_date", "image_rev", "prod_pn", "status"}; + for (auto field : not_parsed_metadata_fields) { + if (!root.isMember(field)) { + logger().warn( + "No " + field + + " found in metadata. Your metadata may be the result " + "of calling to_string() on the sensor_info object OR " + "you recorded this data with a very old version of " + "Ouster Studio. We advise you to record the metadata " + "directly " + "with get_metadata and to update your Ouster Studio."); + } } + sensor_info info{}; // info.name is deprecated, will be empty string if not present info.name = root["hostname"].asString(); + // if these are not present they are also empty strings info.sn = root["prod_sn"].asString(); + info.prod_line = root["prod_line"].asString(); info.fw_rev = root["build_rev"].asString(); + + // checked that lidar_mode is present already - never empty string info.mode = lidar_mode_of_string(root["lidar_mode"].asString()); - info.prod_line = root["prod_line"].asString(); // "data_format" introduced in fw 2.0. Fall back to 1.13 if (root.isMember("data_format")) { info.format = parse_data_format(root["data_format"]); } else { - logger().warn("No data_format found."); + logger().warn("No data_format found. Using default legacy data format"); info.format = default_data_format(info.mode); } // "lidar_origin_to_beam_origin_mm" introduced in fw 2.0 BUT missing - // on OS-DOME. Handle falling back to FW 1.13 or setting to 0 according - // to prod-line + // on OS-DOME. Handle falling back to FW 1.13 or setting to 0 + // according to prod-line if (root.isMember("lidar_origin_to_beam_origin_mm")) { info.lidar_origin_to_beam_origin_mm = root["lidar_origin_to_beam_origin_mm"].asDouble(); @@ -761,9 +861,14 @@ static sensor_info parse_legacy(const std::string& meta) { 0) { // is an OS-DOME - fill with 0 info.lidar_origin_to_beam_origin_mm = 0; } else { // not an OS-DOME - logger().warn("No lidar_origin_to_beam_origin_mm found."); + logger().warn( + "No lidar_origin_to_beam_origin_mm found. Using default " + "value for the specified prod_line or default gen 1 values" + "if prod_line is missing"); info.lidar_origin_to_beam_origin_mm = - default_lidar_origin_to_beam_origin(info.prod_line); + default_lidar_origin_to_beam_origin( + info.prod_line); // note it is possible that + // info.prod_line is "" } } @@ -784,12 +889,11 @@ static sensor_info parse_legacy(const std::string& meta) { } if (root["beam_altitude_angles"].size() != info.format.pixels_per_column) { - throw std::invalid_argument{ - "Unexpected number of beam_altitude_angles"}; + throw std::runtime_error{"Unexpected number of beam_altitude_angles"}; } if (root["beam_azimuth_angles"].size() != info.format.pixels_per_column) { - throw std::invalid_argument{"Unexpected number of beam_azimuth_angles"}; + throw std::runtime_error{"Unexpected number of beam_azimuth_angles"}; } for (const auto& v : root["beam_altitude_angles"]) @@ -809,7 +913,9 @@ static sensor_info parse_legacy(const std::string& meta) { } } } else { - logger().warn("No valid imu_to_sensor_transform found."); + logger().warn( + "No valid imu_to_sensor_transform found. Using default for gen " + "1"); info.imu_to_sensor_transform = default_imu_to_sensor_transform; } @@ -824,10 +930,25 @@ static sensor_info parse_legacy(const std::string& meta) { } } } else { - logger().warn("No valid lidar_to_sensor_transform found."); + logger().warn( + "No valid lidar_to_sensor_transform found. Using default for " + "gen " + "1"); info.lidar_to_sensor_transform = default_lidar_to_sensor_transform; } + auto zero_check = [](auto el, std::string name) { + bool all_zeros = std::all_of(el.cbegin(), el.cend(), + [](double k) { return k == 0.0; }); + if (all_zeros) { + throw std::runtime_error{"Field " + name + + " in the metadata cannot all be zeros."}; + } + }; + + zero_check(info.beam_altitude_angles, "beam_altitude_angles"); + zero_check(info.beam_azimuth_angles, "beam_azimuth_angles"); + info.extrinsic = mat4d::Identity(); // default to 0 if keys are not present @@ -851,7 +972,7 @@ enum configuration_version { FW_2_0 = 3, FW_2_2 = 4 }; std::string convert_to_legacy(const std::string& metadata) { if (!is_new_format(metadata)) throw std::invalid_argument( - "Could not convert invalid non-legacy metadata format"); + "Invalid non-legacy metadata format provided"); Json::Value root{}; Json::CharReaderBuilder read_builder{}; @@ -860,7 +981,7 @@ std::string convert_to_legacy(const std::string& metadata) { if (metadata.size()) { if (!Json::parseFromStream(read_builder, ss, &root, &errors)) { - throw std::invalid_argument{errors.c_str()}; + throw std::runtime_error{errors}; } } Json::Value result{}; @@ -903,12 +1024,12 @@ sensor_info parse_metadata(const std::string& metadata) { if (metadata.size()) { if (!Json::parseFromStream(builder, ss, &root, &errors)) - throw std::invalid_argument{errors.c_str()}; + throw std::runtime_error{errors}; } sensor_info info{}; if (is_new_format(metadata)) { - logger().debug("parsing new metadata format"); + logger().debug("parsing non-legacy metadata format"); info = parse_legacy(convert_to_legacy(metadata)); } else { logger().debug("parsing legacy metadata format"); @@ -927,7 +1048,7 @@ sensor_info metadata_from_json(const std::string& json_file) { if (!ifs) { std::stringstream ss; ss << "Failed to read metadata file: " << json_file; - throw std::runtime_error(ss.str()); + throw std::runtime_error{ss.str()}; } return parse_metadata(buf.str()); @@ -1041,16 +1162,18 @@ Json::Value to_json(const sensor_config& config) { } if (config.signal_multiplier) { - if (config.signal_multiplier < 1) { + check_signal_multiplier(config.signal_multiplier.value()); + if ((config.signal_multiplier == 0.25) || + (config.signal_multiplier == 0.5)) { root["signal_multiplier"] = config.signal_multiplier.value(); } else { - // jsoncpp < 1.7.7 strips 0s off of exact representation so 2.0 - // becomes 2 - // On ubuntu 18.04, the default jsoncpp is 1.7.4-3 - // Fix was: - // https://github.com/open-source-parsers/jsoncpp/pull/547 Work - // around by always casting to int before writing out to json - root["signal_multiplier"] = int(config.signal_multiplier.value()); + // jsoncpp < 1.7.7 strips 0s off of exact representation + // so 2.0 becomes 2 + // On ubuntu 18.04, the default jsoncpp is 1.7.4-3 Fix was: + // https://github.com/open-source-parsers/jsoncpp/pull/547 + // Work around by always casting to int before writing out to json + int signal_multiplier_int = int(config.signal_multiplier.value()); + root["signal_multiplier"] = signal_multiplier_int; } } diff --git a/python/setup.py b/python/setup.py index 42bcc345..c5b70144 100644 --- a/python/setup.py +++ b/python/setup.py @@ -21,6 +21,7 @@ if not os.path.exists(os.path.join(OUSTER_SDK_PATH, "cmake")): raise RuntimeError("Could not guess OUSTER_SDK_PATH") + # https://packaging.python.org/en/latest/guides/single-sourcing-package-version/ def parse_version(): with open(os.path.join(OUSTER_SDK_PATH, 'CMakeLists.txt')) as listfile: @@ -28,6 +29,7 @@ def parse_version(): groups = re.search("set\(OusterSDK_VERSION_STRING ([^-\)]+)(.(.*))?\)", content) return groups.group(1) + (groups.group(3) or "") + class CMakeExtension(Extension): def __init__(self, name, sourcedir=''): Extension.__init__(self, name, sources=[]) diff --git a/python/src/cpp/_client.cpp b/python/src/cpp/_client.cpp index 4acc85bb..e4bbad8a 100644 --- a/python/src/cpp/_client.cpp +++ b/python/src/cpp/_client.cpp @@ -83,7 +83,7 @@ extern const Table multipurpose_io_mode_strings; extern const Table polarity_strings; extern const Table nmea_baud_rate_strings; -extern Table chanfield_strings; +extern Table chanfield_strings; extern Table udp_profile_lidar_strings; extern Table udp_profile_imu_strings; extern Table shot_limiting_status_strings; @@ -298,6 +298,11 @@ PYBIND11_PLUGIN(_client) { .def_readonly("imu_packet_size", &packet_format::imu_packet_size) .def_readonly("columns_per_packet", &packet_format::columns_per_packet) .def_readonly("pixels_per_column", &packet_format::pixels_per_column) + .def_readonly("packet_header_size", &packet_format::packet_header_size) + .def_readonly("col_header_size", &packet_format::col_header_size) + .def_readonly("col_footer_size", &packet_format::col_footer_size) + .def_readonly("col_size", &packet_format::col_size) + .def_readonly("packet_footer_size", &packet_format::packet_footer_size) .def("packet_type", [](packet_format& pf, py::buffer buf) { return pf.packet_type(getptr(pf.lidar_packet_size, buf)); diff --git a/python/src/ouster/client/core.py b/python/src/ouster/client/core.py index 13d3fabc..1566152d 100644 --- a/python/src/ouster/client/core.py +++ b/python/src/ouster/client/core.py @@ -128,9 +128,10 @@ def __init__(self, *, metadata: Optional[SensorInfo] = None, buf_size: int = 128, - timeout: Optional[float] = 1.0, + timeout: Optional[float] = 2.0, _overflow_err: bool = False, _flush_before_read: bool = True, + _flush_frames: int = 5, _legacy_format: bool = True) -> None: """ Neither the ports nor udp destination configuration on the sensor will @@ -157,6 +158,7 @@ def __init__(self, self._flush_before_read = _flush_before_read self._cache = None self._fetched_meta = "" + self._flush_frames = _flush_frames self._legacy_format = _legacy_format # Fetch from sensor if not explicitly provided @@ -240,7 +242,7 @@ def __iter__(self) -> Iterator[Packet]: # Attempt to flush any old data before producing packets if self._flush_before_read: - self.flush(full=True) + self.flush(n_frames = self._flush_frames, full=True) while True: try: @@ -342,7 +344,7 @@ def __init__(self, source: PacketSource, *, complete: bool = False, - timeout: Optional[float] = 1.0, + timeout: Optional[float] = 2.0, fields: Optional[Dict[ChanField, FieldDType]] = None, _max_latency: int = 0) -> None: """ @@ -462,7 +464,7 @@ def next_batch() -> List[LidarScan]: buf_size=n * 128, _flush_before_read=False)) as source: source.flush(full=True) - scans = cls(source, timeout=1.0, complete=True, _max_latency=0) + scans = cls(source, timeout=2.0, complete=True, _max_latency=0) return take(n, scans) return metadata, iter(next_batch, []) @@ -474,7 +476,7 @@ def stream( lidar_port: int = 7502, *, buf_size: int = 640, - timeout: Optional[float] = 1.0, + timeout: Optional[float] = 2.0, complete: bool = True, metadata: Optional[SensorInfo] = None, fields: Optional[Dict[ChanField, FieldDType]] = None) -> 'Scans': diff --git a/python/src/ouster/sdk/viz.py b/python/src/ouster/sdk/viz.py index 48dbf30f..a98235ab 100644 --- a/python/src/ouster/sdk/viz.py +++ b/python/src/ouster/sdk/viz.py @@ -31,7 +31,8 @@ # limit ouster_client log statements to "debug" and direct the output to log file # rather than the console (default). -client.init_logger("info", "ouster-python.log") +# TODO uncomment when we figure out where we want to write it everywhere, have more useful logs +# client.init_logger("info", "ouster-python.log") T = TypeVar('T') diff --git a/python/tests/test_config.py b/python/tests/test_config.py index ad555092..9c2f4744 100644 --- a/python/tests/test_config.py +++ b/python/tests/test_config.py @@ -11,6 +11,13 @@ from ouster import client +# all valid values +valid_signal_multiplier_values = [0.25, 0.5, 1, 2, 3] +# make sure to hit different types of invalid values: +# 0, one double below 1, one double between 1 and 3, +# one int above 3, one double above 3 +invalid_signal_multiplier_values = [0, 0.3, 1.3, 5, 5.5] + @pytest.mark.parametrize("mode, string", [ (client.OperatingMode.OPERATING_NORMAL, "NORMAL"), @@ -319,13 +326,16 @@ def test_copy_config(complete_config_string: str) -> None: def test_parse_config() -> None: """Sanity check parsing from json string.""" - with pytest.raises(ValueError): + with pytest.raises(RuntimeError): client.SensorConfig('/') - with pytest.raises(ValueError): + with pytest.raises(RuntimeError): client.SensorConfig('{ ') -@pytest.mark.parametrize("signal_multiplier", [0.25, 0.5, 1, 2, 3]) +# Note that config.signal_multiplier will also accept values +# that aren't these (but will throw parsing from json or set_config) +# so we don't test the invalid_signal_multiplier_values +@pytest.mark.parametrize("signal_multiplier", valid_signal_multiplier_values) def test_signal_multiplier(signal_multiplier) -> None: """Check that signal multiplier supports all FW values""" @@ -333,6 +343,45 @@ def test_signal_multiplier(signal_multiplier) -> None: config.signal_multiplier = signal_multiplier +def signal_multiplier_config_json(signal_multiplier): + return '{"signal_multiplier": ' + str(signal_multiplier) + '}' + + +@pytest.mark.parametrize("signal_multiplier", valid_signal_multiplier_values) +def test_config_valid_signal_multiplier_parse(signal_multiplier) -> None: + """Check parsing valid signal multiplier from json""" + + client.SensorConfig(signal_multiplier_config_json(signal_multiplier)) + + +@pytest.mark.parametrize("signal_multiplier", invalid_signal_multiplier_values) +def test_config_invalid_signal_multiplier_parse(signal_multiplier) -> None: + """Check parsing invalid signal multiplier from json""" + + with pytest.raises(RuntimeError): + client.SensorConfig(signal_multiplier_config_json(signal_multiplier)) + + +@pytest.mark.parametrize("signal_multiplier", valid_signal_multiplier_values) +def test_config_valid_signal_multiplier_to_str(signal_multiplier) -> None: + """Check writing valid signal multiplier to string""" + + config = client.SensorConfig() + config.signal_multiplier = signal_multiplier + str(config) + + +@pytest.mark.parametrize("signal_multiplier", invalid_signal_multiplier_values) +def test_config_invalid_signal_multiplier_to_str(signal_multiplier) -> None: + """Check writing invalid signal multiplier to string""" + + config = client.SensorConfig() + # note: won't error out here bc it's just a double + config.signal_multiplier = signal_multiplier + with pytest.raises(RuntimeError): + str(config) + + @pytest.fixture() def deprecated_params_config() -> client.SensorConfig: deprecated_params = """ { "udp_ip": "169.254.148.183", "auto_start_flag": 1 }""" diff --git a/python/tests/test_data.py b/python/tests/test_data.py index b4ddaa85..19587965 100644 --- a/python/tests/test_data.py +++ b/python/tests/test_data.py @@ -63,7 +63,7 @@ def test_lidar_packet(meta: client.SensorInfo) -> None: client.UDPProfileLidar.PROFILE_LIDAR_RNG15_RFL8_NIR8) assert len( - client.ChanField.__members__) == 23, "Don't forget to update tests!" + client.ChanField.__members__) == 24, "Don't forget to update tests!" assert np.array_equal(p.field(client.ChanField.RANGE), np.zeros((h, w))) assert np.array_equal(p.field(client.ChanField.REFLECTIVITY), np.zeros((h, w))) diff --git a/python/tests/test_metadata.py b/python/tests/test_metadata.py index 3119648c..09d88e71 100644 --- a/python/tests/test_metadata.py +++ b/python/tests/test_metadata.py @@ -159,21 +159,21 @@ def test_copy_info(meta: client.SensorInfo) -> None: def test_parse_info() -> None: """Sanity check parsing from json.""" - with pytest.raises(ValueError): + with pytest.raises(RuntimeError): client.SensorInfo('/') - with pytest.raises(ValueError): + with pytest.raises(RuntimeError): client.SensorInfo('') - with pytest.raises(ValueError): + with pytest.raises(RuntimeError): client.SensorInfo('{ }') - with pytest.raises(ValueError): + with pytest.raises(RuntimeError): client.SensorInfo('{ "lidar_mode": "1024x10" }') # TODO: this should actually fail unless *all* parameters needed to # unambiguously interpret a sensor data stream are present metadata = { 'lidar_mode': '1024x10', - 'beam_altitude_angles': [0] * 64, - 'beam_azimuth_angles': [0] * 64, + 'beam_altitude_angles': [1] * 64, + 'beam_azimuth_angles': [1] * 64, 'lidar_to_sensor_transform': list(range(16)) } info = client.SensorInfo(json.dumps(metadata)) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0bb921c8..c781cc03 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -19,7 +19,7 @@ set_tests_properties( bcompat_meta_json_test PROPERTIES ENVIRONMENT - DATA_DIR=${CMAKE_CURRENT_LIST_DIR}/metadata) + DATA_DIR=${CMAKE_CURRENT_LIST_DIR}/metadata/) add_executable(lidar_scan_test lidar_scan_test.cpp) @@ -32,3 +32,27 @@ set_tests_properties( ENVIRONMENT DATA_DIR=${CMAKE_CURRENT_LIST_DIR}/metadata/ ) + +add_executable(cartesian_test cartesian_test.cpp) + +target_link_libraries(cartesian_test OusterSDK::ouster_client GTest::gtest GTest::gtest_main) + +add_test(NAME cartesian_test COMMAND cartesian_test --gtest_output=xml:cartesian_test.xml) +set_tests_properties( + cartesian_test + PROPERTIES + ENVIRONMENT + DATA_DIR=${CMAKE_CURRENT_LIST_DIR}/metadata/ +) + +add_executable(metadata_errors_test metadata_errors_test.cpp) + +target_link_libraries(metadata_errors_test OusterSDK::ouster_client GTest::gtest GTest::gtest_main) + +add_test(NAME metadata_errors_test COMMAND metadata_errors_test --gtest_output=xml:metadata_errors_test.xml) +set_tests_properties( + metadata_errors_test + PROPERTIES + ENVIRONMENT + DATA_DIR=${CMAKE_CURRENT_LIST_DIR}/metadata/ +) diff --git a/tests/cartesian_test.cpp b/tests/cartesian_test.cpp new file mode 100644 index 00000000..71edd243 --- /dev/null +++ b/tests/cartesian_test.cpp @@ -0,0 +1,242 @@ +/** + * Copyright (c) 2021, Ouster, Inc. + * All rights reserved. + */ + +#include + +#include +#include +#include +#include + +#include "ouster/lidar_scan.h" + +class Timer { + public: + void start() { t_start = std::chrono::system_clock::now(); } + + void stop() { t_end = std::chrono::system_clock::now(); } + + int64_t elapsed_microseconds() { + using namespace std::chrono; + return duration_cast(t_end - t_start).count(); + } + + private: + std::chrono::time_point t_start; + std::chrono::time_point t_end; +}; + +template +class MovingAverage { + public: + MovingAverage& operator()(T sample) { + total_sum += sample; + if (samples_count < N) + samples[samples_count++] = sample; + else { + T& oldest = samples[samples_count++ % N]; + total_sum -= oldest; + oldest = sample; + } + return *this; + } + + operator double() const { + return static_cast(total_sum) / + static_cast(std::min(samples_count, N)); + } + + private: + T samples[N]; + size_t samples_count{0}; + Total total_sum{0}; +}; + +using namespace ouster; + +// Same as ouster::cartesian but uses single float precision +PointsF cartesian_f(const Eigen::Ref>& range, + const PointsF& direction, const PointsF& offset) { + if (range.cols() * range.rows() != direction.rows()) + throw std::invalid_argument("unexpected image dimensions"); + auto reshaped = Eigen::Map>( + range.data(), range.cols() * range.rows()); + auto nooffset = direction.colwise() * reshaped.cast(); + return (nooffset.array() == 0.0).select(nooffset, nooffset + offset); +} + + +class CarteianParamterizedTestFixture + : public ::testing::TestWithParam> { + protected: + int scan_width; + int scan_height; +}; + +INSTANTIATE_TEST_CASE_P(CarteianParamterizedTests, + CarteianParamterizedTestFixture, + ::testing::Values(std::pair{512, 128}, + std::pair{1024, 128}, + std::pair{2048, 128}, + std::pair{4096, 128})); + +TEST(CarteianParamterizedTestFixture, CartesianFunctionsMatch) { + const auto WIDTH = 512; + const auto HEIGHT = 64; + const auto ROWS = WIDTH * HEIGHT; + const auto COLS = 3; + + PointsD direction = + 0.5 * PointsD::Random(ROWS, COLS) + PointsD::Constant(ROWS, COLS, 1.0); + PointsD offset = 0.005 * (PointsD::Random(ROWS, COLS) + + PointsD::Constant(ROWS, COLS, 1.0)); + XYZLut lut{direction, offset}; + + img_t range = img_t::Random(WIDTH, HEIGHT); + + auto points0 = cartesian(range, lut); + + PointsD points = PointsD::Zero(ROWS, COLS); + + cartesianT(points, range, direction, offset); + EXPECT_TRUE(points.isApprox(points0)); +} + +TEST(CarteianParamterizedTestFixture, CartesianFunctionsMatchF) { + const auto WIDTH = 512; + const auto HEIGHT = 64; + const auto ROWS = WIDTH * HEIGHT; + const auto COLS = 3; + + PointsD direction = + 0.5 * PointsD::Random(ROWS, COLS) + PointsD::Constant(ROWS, COLS, 1.0); + PointsD offset = 0.005 * (PointsD::Random(ROWS, COLS) + + PointsD::Constant(ROWS, COLS, 1.0)); + XYZLut lut{direction, offset}; + + PointsF directionF = direction.cast(); + PointsF offsetF = offset.cast(); + + img_t range = img_t::Random(WIDTH, HEIGHT); + + auto points0 = cartesian(range, lut); + auto points0F = points0.cast(); + + PointsF pointsF = PointsF::Zero(ROWS, COLS); + + cartesianT(pointsF, range, directionF, offsetF); + EXPECT_TRUE(pointsF.isApprox(points0F)); +} + +TEST_P(CarteianParamterizedTestFixture, SpeedCheck) { + std::map styles = { + {"red", "\033[0;31m"}, {"green", "\033[0;32m"}, + {"yellow", "\033[0;33m"}, {"blue", "\033[0;34m"}, + {"magenta", "\033[0;35m"}, {"cyan", "\033[0;36m"}, + {"bold", "\033[1m"}, {"reset", "\033[0m"}}; + + const auto test_params = GetParam(); + const auto WIDTH = test_params.first; + const auto HEIGHT = test_params.second; + const auto ROWS = WIDTH * HEIGHT; + const auto COLS = 3; + std::cout << styles["yellow"] << styles["bold"] + << "CHECKING PERFORMANCE FOR LIDAR MODE: [" << WIDTH << "x" + << HEIGHT << "]" << styles["reset"] << std::endl; + + PointsD direction = + 0.5 * PointsD::Random(ROWS, COLS) + PointsD::Constant(ROWS, COLS, 1.0); + PointsD offset = 0.005 * (PointsD::Random(ROWS, COLS) + + PointsD::Constant(ROWS, COLS, 1.0)); + XYZLut lut{direction, offset}; + + PointsF directionF = direction.cast(); + PointsF offsetF = offset.cast(); + + // create an empty arrays of points + PointsD points = PointsD(ROWS, COLS); + PointsF pointsF = PointsF(ROWS, COLS); + img_t range = img_t(WIDTH, HEIGHT); + + constexpr int N_SCANS = 1000; + constexpr int MOVING_AVG_WINDOW = 100; + using MovingAverage64 = MovingAverage; + static std::map mv; + + Timer t; + std::stringstream ss; + int output_ctr = 0; + + using CartesianMethod = std::function& range)>; + std::vector> all_cartesians; + + all_cartesians.emplace_back("c0", [&](const img_t& range) { + points = cartesian(range, lut); + }); + + all_cartesians.emplace_back("c0f", [&](const img_t& range) { + pointsF = cartesian_f(range, directionF, offsetF); + }); + + all_cartesians.emplace_back("cT", [&](const img_t& range) { + cartesianT(points, range, direction, offset); + }); + + all_cartesians.emplace_back("cfT", [&](const img_t& range) { + cartesianT(pointsF, range, directionF, offsetF); + }); + + std::default_random_engine g; + std::uniform_real_distribution d(0.0, 1.0); + std::vector ids(all_cartesians.size()); + std::iota(std::begin(ids), std::end(ids), 0); + + for (auto i = 0; i < N_SCANS; ++i) { + auto p = std::ceil(10 * float(i) / float(N_SCANS)) / 10; + auto unary_expr = [&g, &d, p](auto) { + return d(g) >= p ? 0U : static_cast(d(g) * 10000); + }; + + auto valid_returns = (range != 0).count(); + auto percentage_valid = + int(roundf(float(valid_returns) / float(range.size()) * 100)); + + std::shuffle(std::begin(ids), std::end(ids), g); + for (auto i : ids) { + const auto& method = all_cartesians[i]; + range = range.unaryExpr(unary_expr); + t.start(); + method.second(range); + t.stop(); + mv[method.first](t.elapsed_microseconds()); + } + + if (++output_ctr % MOVING_AVG_WINDOW == 0) { + ss.str(""); + ss << styles["bold"] << "returns: " << styles["reset"] + << styles["magenta"] << std::setw(3) << percentage_valid << "%, " + << styles["reset"]; + ss << styles["bold"] << "c0[time]: " << styles["reset"] + << styles["cyan"] << std::setw(4) << mv["c0"] << "μs, " + << styles["reset"]; + + auto best_time = std::min_element( + mv.begin(), mv.end(), [](const auto& a, const auto& b) -> bool { + return a.second < b.second; + }); + + for (const auto& x : mv) { + auto speedup = lround(100.0f * mv["c0"] / x.second); + auto color_modifier = + x.first == best_time->first + ? styles["blue"] + : (speedup >= 100 ? styles["green"] : styles["red"]); + ss << styles["bold"] << x.first << ": " << color_modifier + << std::setw(4) << speedup << "%, " << styles["reset"]; + } + std::cout << ss.str() << std::endl; + } + } +} diff --git a/tests/metadata/malformed/complete_but_all_zeros_legacy.json b/tests/metadata/malformed/complete_but_all_zeros_legacy.json new file mode 100644 index 00000000..9d136218 --- /dev/null +++ b/tests/metadata/malformed/complete_but_all_zeros_legacy.json @@ -0,0 +1,480 @@ +{ + "beam_altitude_angles": + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "beam_azimuth_angles": + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "beam_to_lidar_transform": + [ + 1, + 0, + 0, + 27.67, + 0, + 1, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 1 + ], + "build_date": "2022-10-19T08:50:01Z", + "build_rev": "v2.4.0-259-g648711140", + "client_version": "ouster_client 0.5.3dev1", + "data_format": + { + "column_window": + [ + 0, + 1023 + ], + "columns_per_frame": 1024, + "columns_per_packet": 16, + "pixel_shift_by_row": + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "pixels_per_column": 128, + "udp_profile_imu": "LEGACY", + "udp_profile_lidar": "LEGACY" + }, + "hostname": "", + "image_rev": "ousteros-image-prod-bootes-v3.0.0-alpha.4+20221019084911.staging-2022.ci-git.master.648711140d", + "imu_to_sensor_transform": + [ + 1, + 0, + 0, + 6.253, + 0, + 1, + 0, + -11.775, + 0, + 0, + 1, + 7.645, + 0, + 0, + 0, + 1 + ], + "initialization_id": 1629352, + "json_calibration_version": 4, + "lidar_mode": "1024x10", + "lidar_origin_to_beam_origin_mm": 27.67, + "lidar_to_sensor_transform": + [ + -1, + 0, + 0, + 0, + 0, + -1, + 0, + 0, + 0, + 0, + 1, + 36.18, + 0, + 0, + 0, + 1 + ], + "prod_line": "OS-0-128", + "prod_pn": "840-104791-A", + "prod_sn": "992242000335", + "status": "RUNNING", + "udp_port_imu": 7503, + "udp_port_lidar": 7502 +} \ No newline at end of file diff --git a/tests/metadata/malformed/garbled_legacy_and_nonlegacy.json b/tests/metadata/malformed/garbled_legacy_and_nonlegacy.json new file mode 100644 index 00000000..0e7c90e7 --- /dev/null +++ b/tests/metadata/malformed/garbled_legacy_and_nonlegacy.json @@ -0,0 +1,268 @@ +{ + "beam_intrinsics": + { + "beam_altitude_angles": + [ + 16.729, + 16.118, + 15.543, + 14.995, + 14.526, + 13.937, + 13.381, + 12.837, + 12.378, + 11.801, + 11.251, + 10.704, + 10.251, + 9.693, + 9.138, + 8.603, + 8.149, + 7.594, + 7.049, + 6.513, + 6.056, + 5.504, + 4.961, + 4.419, + 3.967, + 3.424, + 2.881, + 2.328, + 1.88, + 1.337, + 0.787, + 0.239, + -0.205, + -0.756, + -1.39, + -1.854, + -2.3, + -2.839, + -3.386, + -3.935, + -4.391, + -4.929, + -5.472, + -6.028, + -6.471, + -7.02, + -7.563, + -8.116, + -8.572, + -9.112, + -9.66, + -10.227, + -10.687, + -11.226, + -11.777, + -12.35, + -12.807, + -13.347, + -13.902, + -14.49, + -14.968, + -15.504, + -16.078, + -16.679 + ], + "beam_azimuth_angles": + [ + 3.073, + 0.904, + -1.261, + -3.384, + 3.048, + 0.9, + -1.235, + -3.335, + 3.029, + 0.915, + -1.2, + -3.301, + 3.026, + 0.921, + -1.179, + -3.268, + 3.032, + 0.933, + -1.156, + -3.23, + 3.028, + 0.953, + -1.131, + -3.209, + 3.059, + 0.976, + -1.112, + -3.182, + 3.071, + 0.99, + -1.087, + -3.155, + 3.097, + 1.014, + -1.103, + -3.133, + 3.118, + 1.035, + -1.039, + -3.116, + 3.144, + 1.061, + -1.013, + -3.101, + 3.178, + 1.092, + -0.993, + -3.089, + 3.217, + 1.125, + -0.972, + -3.072, + 3.265, + 1.159, + -0.949, + -3.066, + 3.321, + 1.204, + -0.921, + -3.055, + 3.381, + 1.252, + -0.898, + -3.071 + ], + "lidar_origin_to_beam_origin_mm": 12.163 + }, + "calibration_status": "error: Command not recognized.", + "client_version": "ouster_client 0.2.2dev0", + "imu_to_sensor_transform": + [ + 1, + 0, + 0, + 6.253, + 0, + 1, + 0, + -11.775, + 0, + 0, + 1, + 7.645, + 0, + 0, + 0, + 1 + ], + "data_format": + { + "column_window": + [ + 0, + 1023 + ], + "columns_per_frame": 1024, + "columns_per_packet": 16, + "pixel_shift_by_row": + [ + 19, + 13, + 6, + 0, + 19, + 13, + 6, + 1, + 19, + 13, + 7, + 1, + 19, + 13, + 7, + 1, + 19, + 13, + 7, + 1, + 19, + 13, + 7, + 1, + 19, + 13, + 7, + 1, + 19, + 13, + 7, + 1, + 19, + 13, + 7, + 1, + 19, + 13, + 7, + 1, + 19, + 13, + 7, + 1, + 19, + 13, + 7, + 1, + 19, + 13, + 7, + 1, + 19, + 13, + 7, + 1, + 19, + 13, + 7, + 1, + 20, + 14, + 7, + 1 + ], + "pixels_per_column": 64 + }, + "lidar_intrinsics": + { + "lidar_to_sensor_transform": + [ + -1, + 0, + 0, + 0, + 0, + -1, + 0, + 0, + 0, + 0, + 1, + 36.18, + 0, + 0, + 0, + 1 + ] + }, + "build_date": "2020-11-24T06:51:26Z", + "build_rev": "v2.0.0", + "image_rev": "ousteros-image-prod-aries-v2.0.0+20201124065024", + "prod_line": "OS-1-64", + "prod_pn": "840-101396-03", + "prod_sn": "991913000010", + "status": "RUNNING" +} diff --git a/tests/metadata/malformed/incomplete_data_format_legacy.json b/tests/metadata/malformed/incomplete_data_format_legacy.json new file mode 100644 index 00000000..6ad27408 --- /dev/null +++ b/tests/metadata/malformed/incomplete_data_format_legacy.json @@ -0,0 +1,460 @@ +{ + "beam_altitude_angles": + [ + 21.34, + 21.03, + 20.72, + 20.41, + 20.08, + 19.78, + 19.48, + 19.17, + 18.83, + 18.51, + 18.21, + 17.88, + 17.56, + 17.23, + 16.92, + 16.59, + 16.26, + 15.94, + 15.61, + 15.28, + 14.93, + 14.6, + 14.27, + 13.95, + 13.61, + 13.26, + 12.94, + 12.6, + 12.26, + 11.91, + 11.58, + 11.24, + 10.89, + 10.55, + 10.21, + 9.88, + 9.52, + 9.18, + 8.83, + 8.5, + 8.14, + 7.78, + 7.46, + 7.11, + 6.75, + 6.39, + 6.05, + 5.7, + 5.33, + 4.99, + 4.64, + 4.29, + 3.94, + 3.59, + 3.24, + 2.88, + 2.52, + 2.18, + 1.81, + 1.48, + 1.11, + 0.76, + 0.4, + 0.05, + -0.3, + -0.67, + -1.01, + -1.37, + -1.75, + -2.08, + -2.43, + -2.79, + -3.15, + -3.5, + -3.86, + -4.2, + -4.56, + -4.9, + -5.27, + -5.6, + -5.96, + -6.32, + -6.68, + -7, + -7.38, + -7.71, + -8.05, + -8.41, + -8.77, + -9.11, + -9.45, + -9.79, + -10.16, + -10.49, + -10.83, + -11.18, + -11.54, + -11.86, + -12.2, + -12.55, + -12.91, + -13.24, + -13.58, + -13.89, + -14.26, + -14.58, + -14.89, + -15.21, + -15.6, + -15.89, + -16.22, + -16.54, + -16.88, + -17.21, + -17.52, + -17.85, + -18.2, + -18.5, + -18.82, + -19.14, + -19.47, + -19.78, + -20.09, + -20.38, + -20.72, + -21.03, + -21.33, + -21.62 + ], + "beam_azimuth_angles": + [ + 4.27, + 1.43, + -1.39, + -4.23, + 4.26, + 1.45, + -1.38, + -4.22, + 4.28, + 1.43, + -1.39, + -4.24, + 4.27, + 1.44, + -1.4, + -4.22, + 4.28, + 1.45, + -1.39, + -4.23, + 4.26, + 1.44, + -1.4, + -4.23, + 4.27, + 1.43, + -1.39, + -4.23, + 4.26, + 1.43, + -1.4, + -4.24, + 4.26, + 1.44, + -1.4, + -4.23, + 4.28, + 1.43, + -1.4, + -4.24, + 4.26, + 1.42, + -1.39, + -4.24, + 4.27, + 1.43, + -1.42, + -4.25, + 4.26, + 1.43, + -1.42, + -4.25, + 4.28, + 1.43, + -1.42, + -4.25, + 4.26, + 1.43, + -1.44, + -4.25, + 4.27, + 1.43, + -1.43, + -4.26, + 4.27, + 1.42, + -1.43, + -4.26, + 4.25, + 1.42, + -1.43, + -4.28, + 4.25, + 1.42, + -1.45, + -4.28, + 4.26, + 1.42, + -1.44, + -4.26, + 4.26, + 1.4, + -1.46, + -4.27, + 4.24, + 1.41, + -1.43, + -4.28, + 4.26, + 1.4, + -1.44, + -4.28, + 4.22, + 1.4, + -1.43, + -4.29, + 4.25, + 1.41, + -1.45, + -4.29, + 4.24, + 1.4, + -1.46, + -4.28, + 4.24, + 1.4, + -1.44, + -4.28, + 4.22, + 1.41, + -1.45, + -4.28, + 4.25, + 1.39, + -1.45, + -4.29, + 4.23, + 1.4, + -1.45, + -4.3, + 4.22, + 1.39, + -1.47, + -4.29, + 4.22, + 1.38, + -1.47, + -4.3 + ], + "build_date": "2022-09-21T17:47:45Z", + "build_rev": "v2.4.0", + "client_version": "ouster_client 0.5.3dev3", + "data_format": + { + "column_window": + [ + 0, + 1023 + ], + "columns_per_packet": 16, + "pixel_shift_by_row": + [ + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12 + ], + "pixels_per_column": 128, + "udp_profile_imu": "LEGACY", + "udp_profile_lidar": "RNG19_RFL8_SIG16_NIR16" + }, + "hostname": "", + "image_rev": "ousteros-image-prod-aries-v2.4.0+20220921174636", + "imu_to_sensor_transform": + [ + 1, + 0, + 0, + 6.253, + 0, + 1, + 0, + -11.775, + 0, + 0, + 1, + 7.645, + 0, + 0, + 0, + 1 + ], + "initialization_id": 2573180, + "json_calibration_version": 4, + "lidar_mode": "1024x10", + "lidar_origin_to_beam_origin_mm": 15.806, + "lidar_to_sensor_transform": + [ + -1, + 0, + 0, + 0, + 0, + -1, + 0, + 0, + 0, + 0, + 1, + 36.18, + 0, + 0, + 0, + 1 + ], + "prod_line": "OS-1-128", + "prod_pn": "840-103575-06", + "prod_sn": "992146000760", + "status": "RUNNING", + "udp_port_imu": 7503, + "udp_port_lidar": 7502 +} diff --git a/tests/metadata/malformed/incomplete_data_format_nonlegacy.json b/tests/metadata/malformed/incomplete_data_format_nonlegacy.json new file mode 100644 index 00000000..3e80d7c6 --- /dev/null +++ b/tests/metadata/malformed/incomplete_data_format_nonlegacy.json @@ -0,0 +1,505 @@ +{ + "beam_intrinsics": + { + "beam_altitude_angles": + [ + 21.34, + 21.03, + 20.72, + 20.41, + 20.08, + 19.78, + 19.48, + 19.17, + 18.83, + 18.51, + 18.21, + 17.88, + 17.56, + 17.23, + 16.92, + 16.59, + 16.26, + 15.94, + 15.61, + 15.28, + 14.93, + 14.6, + 14.27, + 13.95, + 13.61, + 13.26, + 12.94, + 12.6, + 12.26, + 11.91, + 11.58, + 11.24, + 10.89, + 10.55, + 10.21, + 9.88, + 9.52, + 9.18, + 8.83, + 8.5, + 8.14, + 7.78, + 7.46, + 7.11, + 6.75, + 6.39, + 6.05, + 5.7, + 5.33, + 4.99, + 4.64, + 4.29, + 3.94, + 3.59, + 3.24, + 2.88, + 2.52, + 2.18, + 1.81, + 1.48, + 1.11, + 0.76, + 0.4, + 0.05, + -0.3, + -0.67, + -1.01, + -1.37, + -1.75, + -2.08, + -2.43, + -2.79, + -3.15, + -3.5, + -3.86, + -4.2, + -4.56, + -4.9, + -5.27, + -5.6, + -5.96, + -6.32, + -6.68, + -7, + -7.38, + -7.71, + -8.05, + -8.41, + -8.77, + -9.11, + -9.45, + -9.79, + -10.16, + -10.49, + -10.83, + -11.18, + -11.54, + -11.86, + -12.2, + -12.55, + -12.91, + -13.24, + -13.58, + -13.89, + -14.26, + -14.58, + -14.89, + -15.21, + -15.6, + -15.89, + -16.22, + -16.54, + -16.88, + -17.21, + -17.52, + -17.85, + -18.2, + -18.5, + -18.82, + -19.14, + -19.47, + -19.78, + -20.09, + -20.38, + -20.72, + -21.03, + -21.33, + -21.62 + ], + "beam_azimuth_angles": + [ + 4.27, + 1.43, + -1.39, + -4.23, + 4.26, + 1.45, + -1.38, + -4.22, + 4.28, + 1.43, + -1.39, + -4.24, + 4.27, + 1.44, + -1.4, + -4.22, + 4.28, + 1.45, + -1.39, + -4.23, + 4.26, + 1.44, + -1.4, + -4.23, + 4.27, + 1.43, + -1.39, + -4.23, + 4.26, + 1.43, + -1.4, + -4.24, + 4.26, + 1.44, + -1.4, + -4.23, + 4.28, + 1.43, + -1.4, + -4.24, + 4.26, + 1.42, + -1.39, + -4.24, + 4.27, + 1.43, + -1.42, + -4.25, + 4.26, + 1.43, + -1.42, + -4.25, + 4.28, + 1.43, + -1.42, + -4.25, + 4.26, + 1.43, + -1.44, + -4.25, + 4.27, + 1.43, + -1.43, + -4.26, + 4.27, + 1.42, + -1.43, + -4.26, + 4.25, + 1.42, + -1.43, + -4.28, + 4.25, + 1.42, + -1.45, + -4.28, + 4.26, + 1.42, + -1.44, + -4.26, + 4.26, + 1.4, + -1.46, + -4.27, + 4.24, + 1.41, + -1.43, + -4.28, + 4.26, + 1.4, + -1.44, + -4.28, + 4.22, + 1.4, + -1.43, + -4.29, + 4.25, + 1.41, + -1.45, + -4.29, + 4.24, + 1.4, + -1.46, + -4.28, + 4.24, + 1.4, + -1.44, + -4.28, + 4.22, + 1.41, + -1.45, + -4.28, + 4.25, + 1.39, + -1.45, + -4.29, + 4.23, + 1.4, + -1.45, + -4.3, + 4.22, + 1.39, + -1.47, + -4.29, + 4.22, + 1.38, + -1.47, + -4.3 + ], + "lidar_origin_to_beam_origin_mm": 15.806 + }, + "calibration_status": + { + "reflectivity": + { + "timestamp": "2021-11-23T05:37:45", + "valid": true + } + }, + "client_version": "ouster_client 0.5.3dev3", + "config_params": + { + "azimuth_window": + [ + 0, + 360000 + ], + "columns_per_packet": 16, + "lidar_mode": "1024x10", + "multipurpose_io_mode": "INPUT_NMEA_UART", + "nmea_baud_rate": "BAUD_115200", + "nmea_ignore_valid_char": 0, + "nmea_in_polarity": "ACTIVE_LOW", + "nmea_leap_seconds": 0, + "operating_mode": "NORMAL", + "phase_lock_enable": false, + "phase_lock_offset": 0, + "signal_multiplier": 1, + "sync_pulse_in_polarity": "ACTIVE_LOW", + "sync_pulse_out_angle": 180, + "sync_pulse_out_frequency": 10, + "sync_pulse_out_polarity": "ACTIVE_HIGH", + "sync_pulse_out_pulse_width": 10, + "timestamp_mode": "TIME_FROM_SYNC_PULSE_IN", + "udp_dest": "10.0.0.167", + "udp_port_imu": 7503, + "udp_port_lidar": 7502, + "udp_profile_imu": "LEGACY", + "udp_profile_lidar": "RNG19_RFL8_SIG16_NIR16" + }, + "imu_intrinsics": + { + "imu_to_sensor_transform": + [ + 1, + 0, + 0, + 6.253, + 0, + 1, + 0, + -11.775, + 0, + 0, + 1, + 7.645, + 0, + 0, + 0, + 1 + ] + }, + "lidar_data_format": + { + "column_window": + [ + 0, + 1023 + ], + "columns_per_frame": 1024, + "columns_per_packet": 16, + "pixel_shift_by_row": + [ + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12 + ], + "udp_profile_imu": "LEGACY", + "udp_profile_lidar": "RNG19_RFL8_SIG16_NIR16" + }, + "lidar_intrinsics": + { + "lidar_to_sensor_transform": + [ + -1, + 0, + 0, + 0, + 0, + -1, + 0, + 0, + 0, + 0, + 1, + 36.18, + 0, + 0, + 0, + 1 + ] + }, + "sensor_info": + { + "build_date": "2022-09-21T17:47:45Z", + "build_rev": "v2.4.0", + "image_rev": "ousteros-image-prod-aries-v2.4.0+20220921174636", + "initialization_id": 2573180, + "prod_line": "OS-1-128", + "prod_pn": "840-103575-06", + "prod_sn": "992146000760", + "status": "RUNNING" + } +} diff --git a/tests/metadata/malformed/incomplete_no_calref_nonlegacy.json b/tests/metadata/malformed/incomplete_no_calref_nonlegacy.json new file mode 100644 index 00000000..bfb6a0ac --- /dev/null +++ b/tests/metadata/malformed/incomplete_no_calref_nonlegacy.json @@ -0,0 +1,503 @@ +{ + "beam_intrinsics": + { + "beam_altitude_angles": + [ + 21.51, + 21.18, + 20.87, + 20.57, + 20.26, + 19.94, + 19.63, + 19.32, + 19.01, + 18.68, + 18.35, + 18.05, + 17.72, + 17.38, + 17.06, + 16.74, + 16.42, + 16.09, + 15.76, + 15.44, + 15.09, + 14.77, + 14.43, + 14.1, + 13.75, + 13.42, + 13.1, + 12.77, + 12.41, + 12.06, + 11.73, + 11.4, + 11.04, + 10.71, + 10.36, + 10.03, + 9.67, + 9.32, + 9.01, + 8.65, + 8.28, + 7.94, + 7.59, + 7.26, + 6.91, + 6.53, + 6.21, + 5.87, + 5.5, + 5.15, + 4.8, + 4.46, + 4.11, + 3.74, + 3.39, + 3.06, + 2.68, + 2.34, + 1.99, + 1.62, + 1.27, + 0.92, + 0.58, + 0.21, + -0.14, + -0.5, + -0.86, + -1.18, + -1.55, + -1.91, + -2.25, + -2.59, + -2.98, + -3.33, + -3.68, + -4.01, + -4.38, + -4.74, + -5.08, + -5.44, + -5.8, + -6.14, + -6.48, + -6.83, + -7.19, + -7.54, + -7.88, + -8.22, + -8.59, + -8.94, + -9.26, + -9.63, + -9.97, + -10.3, + -10.65, + -10.97, + -11.33, + -11.68, + -12.02, + -12.36, + -12.72, + -13.06, + -13.37, + -13.7, + -14.05, + -14.39, + -14.71, + -15.06, + -15.39, + -15.72, + -16.04, + -16.38, + -16.7, + -17.03, + -17.34, + -17.67, + -18.01, + -18.33, + -18.64, + -18.93, + -19.29, + -19.59, + -19.89, + -20.2, + -20.51, + -20.82, + -21.14, + -21.44 + ], + "beam_azimuth_angles": + [ + 4.26, + 1.44, + -1.39, + -4.22, + 4.27, + 1.45, + -1.38, + -4.22, + 4.26, + 1.45, + -1.38, + -4.2, + 4.25, + 1.43, + -1.4, + -4.22, + 4.28, + 1.44, + -1.4, + -4.22, + 4.27, + 1.45, + -1.4, + -4.22, + 4.27, + 1.44, + -1.39, + -4.24, + 4.26, + 1.43, + -1.39, + -4.24, + 4.26, + 1.44, + -1.4, + -4.24, + 4.25, + 1.4, + -1.4, + -4.25, + 4.25, + 1.44, + -1.41, + -4.25, + 4.28, + 1.42, + -1.4, + -4.24, + 4.26, + 1.41, + -1.43, + -4.25, + 4.26, + 1.41, + -1.42, + -4.24, + 4.24, + 1.43, + -1.42, + -4.26, + 4.27, + 1.42, + -1.42, + -4.27, + 4.26, + 1.41, + -1.45, + -4.24, + 4.24, + 1.42, + -1.41, + -4.24, + 4.25, + 1.42, + -1.43, + -4.25, + 4.27, + 1.39, + -1.43, + -4.26, + 4.23, + 1.39, + -1.44, + -4.27, + 4.26, + 1.41, + -1.43, + -4.26, + 4.24, + 1.4, + -1.42, + -4.28, + 4.24, + 1.42, + -1.43, + -4.26, + 4.24, + 1.4, + -1.43, + -4.26, + 4.22, + 1.4, + -1.43, + -4.27, + 4.24, + 1.41, + -1.44, + -4.28, + 4.24, + 1.4, + -1.45, + -4.29, + 4.23, + 1.39, + -1.45, + -4.27, + 4.22, + 1.38, + -1.45, + -4.27, + 4.2, + 1.39, + -1.45, + -4.29, + 4.22, + 1.39, + -1.45, + -4.29 + ], + "lidar_origin_to_beam_origin_mm": 15.806 + }, + "client_version": "ouster_client 0.3.0", + "config_params": + { + "auto_start_flag": 1, + "azimuth_window": + [ + 0, + 360000 + ], + "columns_per_packet": 16, + "lidar_mode": "1024x10", + "multipurpose_io_mode": "OFF", + "nmea_baud_rate": "BAUD_9600", + "nmea_ignore_valid_char": 0, + "nmea_in_polarity": "ACTIVE_HIGH", + "nmea_leap_seconds": 0, + "operating_mode": "NORMAL", + "phase_lock_enable": false, + "phase_lock_offset": 0, + "signal_multiplier": 1, + "sync_pulse_in_polarity": "ACTIVE_HIGH", + "sync_pulse_out_angle": 360, + "sync_pulse_out_frequency": 1, + "sync_pulse_out_polarity": "ACTIVE_HIGH", + "sync_pulse_out_pulse_width": 10, + "timestamp_mode": "TIME_FROM_INTERNAL_OSC", + "udp_dest": "192.168.88.254", + "udp_ip": "192.168.88.254", + "udp_port_imu": 7503, + "udp_port_lidar": 7502, + "udp_profile_imu": "LEGACY", + "udp_profile_lidar": "LEGACY" + }, + "imu_intrinsics": + { + "imu_to_sensor_transform": + [ + 1, + 0, + 0, + 6.253, + 0, + 1, + 0, + -11.775, + 0, + 0, + 1, + 7.645, + 0, + 0, + 0, + 1 + ] + }, + "lidar_data_format": + { + "column_window": + [ + 0, + 1023 + ], + "columns_per_frame": 1024, + "columns_per_packet": 16, + "pixel_shift_by_row": + [ + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0 + ], + "pixels_per_column": 128, + "udp_profile_imu": "LEGACY", + "udp_profile_lidar": "LEGACY" + }, + "lidar_intrinsics": + { + "lidar_to_sensor_transform": + [ + -1, + 0, + 0, + 0, + 0, + -1, + 0, + 0, + 0, + 0, + 1, + 36.18, + 0, + 0, + 0, + 1 + ] + }, + "sensor_info": + { + "base_pn": "", + "base_sn": "", + "build_date": "2021-09-30T15:58:18Z", + "build_rev": "v2.2.0-rc.1", + "image_rev": "ousteros-image-prod-aries-v2.2.0-rc.1", + "initialization_id": 7109745, + "prod_line": "OS-1-128", + "prod_pn": "840-102145-05", + "prod_sn": "992119000444", + "proto_rev": "", + "status": "RUNNING" + } +} diff --git a/tests/metadata/malformed/incomplete_no_sensor_info_nonlegacy.json b/tests/metadata/malformed/incomplete_no_sensor_info_nonlegacy.json new file mode 100644 index 00000000..d203f1a3 --- /dev/null +++ b/tests/metadata/malformed/incomplete_no_sensor_info_nonlegacy.json @@ -0,0 +1,495 @@ +{ + "beam_intrinsics": + { + "beam_altitude_angles": + [ + 21.34, + 21.03, + 20.72, + 20.41, + 20.08, + 19.78, + 19.48, + 19.17, + 18.83, + 18.51, + 18.21, + 17.88, + 17.56, + 17.23, + 16.92, + 16.59, + 16.26, + 15.94, + 15.61, + 15.28, + 14.93, + 14.6, + 14.27, + 13.95, + 13.61, + 13.26, + 12.94, + 12.6, + 12.26, + 11.91, + 11.58, + 11.24, + 10.89, + 10.55, + 10.21, + 9.88, + 9.52, + 9.18, + 8.83, + 8.5, + 8.14, + 7.78, + 7.46, + 7.11, + 6.75, + 6.39, + 6.05, + 5.7, + 5.33, + 4.99, + 4.64, + 4.29, + 3.94, + 3.59, + 3.24, + 2.88, + 2.52, + 2.18, + 1.81, + 1.48, + 1.11, + 0.76, + 0.4, + 0.05, + -0.3, + -0.67, + -1.01, + -1.37, + -1.75, + -2.08, + -2.43, + -2.79, + -3.15, + -3.5, + -3.86, + -4.2, + -4.56, + -4.9, + -5.27, + -5.6, + -5.96, + -6.32, + -6.68, + -7, + -7.38, + -7.71, + -8.05, + -8.41, + -8.77, + -9.11, + -9.45, + -9.79, + -10.16, + -10.49, + -10.83, + -11.18, + -11.54, + -11.86, + -12.2, + -12.55, + -12.91, + -13.24, + -13.58, + -13.89, + -14.26, + -14.58, + -14.89, + -15.21, + -15.6, + -15.89, + -16.22, + -16.54, + -16.88, + -17.21, + -17.52, + -17.85, + -18.2, + -18.5, + -18.82, + -19.14, + -19.47, + -19.78, + -20.09, + -20.38, + -20.72, + -21.03, + -21.33, + -21.62 + ], + "beam_azimuth_angles": + [ + 4.27, + 1.43, + -1.39, + -4.23, + 4.26, + 1.45, + -1.38, + -4.22, + 4.28, + 1.43, + -1.39, + -4.24, + 4.27, + 1.44, + -1.4, + -4.22, + 4.28, + 1.45, + -1.39, + -4.23, + 4.26, + 1.44, + -1.4, + -4.23, + 4.27, + 1.43, + -1.39, + -4.23, + 4.26, + 1.43, + -1.4, + -4.24, + 4.26, + 1.44, + -1.4, + -4.23, + 4.28, + 1.43, + -1.4, + -4.24, + 4.26, + 1.42, + -1.39, + -4.24, + 4.27, + 1.43, + -1.42, + -4.25, + 4.26, + 1.43, + -1.42, + -4.25, + 4.28, + 1.43, + -1.42, + -4.25, + 4.26, + 1.43, + -1.44, + -4.25, + 4.27, + 1.43, + -1.43, + -4.26, + 4.27, + 1.42, + -1.43, + -4.26, + 4.25, + 1.42, + -1.43, + -4.28, + 4.25, + 1.42, + -1.45, + -4.28, + 4.26, + 1.42, + -1.44, + -4.26, + 4.26, + 1.4, + -1.46, + -4.27, + 4.24, + 1.41, + -1.43, + -4.28, + 4.26, + 1.4, + -1.44, + -4.28, + 4.22, + 1.4, + -1.43, + -4.29, + 4.25, + 1.41, + -1.45, + -4.29, + 4.24, + 1.4, + -1.46, + -4.28, + 4.24, + 1.4, + -1.44, + -4.28, + 4.22, + 1.41, + -1.45, + -4.28, + 4.25, + 1.39, + -1.45, + -4.29, + 4.23, + 1.4, + -1.45, + -4.3, + 4.22, + 1.39, + -1.47, + -4.29, + 4.22, + 1.38, + -1.47, + -4.3 + ], + "lidar_origin_to_beam_origin_mm": 15.806 + }, + "calibration_status": + { + "reflectivity": + { + "timestamp": "2021-11-23T05:37:45", + "valid": true + } + }, + "client_version": "ouster_client 0.5.3dev3", + "config_params": + { + "azimuth_window": + [ + 0, + 360000 + ], + "columns_per_packet": 16, + "lidar_mode": "1024x10", + "multipurpose_io_mode": "INPUT_NMEA_UART", + "nmea_baud_rate": "BAUD_115200", + "nmea_ignore_valid_char": 0, + "nmea_in_polarity": "ACTIVE_LOW", + "nmea_leap_seconds": 0, + "operating_mode": "NORMAL", + "phase_lock_enable": false, + "phase_lock_offset": 0, + "signal_multiplier": 1, + "sync_pulse_in_polarity": "ACTIVE_LOW", + "sync_pulse_out_angle": 180, + "sync_pulse_out_frequency": 10, + "sync_pulse_out_polarity": "ACTIVE_HIGH", + "sync_pulse_out_pulse_width": 10, + "timestamp_mode": "TIME_FROM_SYNC_PULSE_IN", + "udp_dest": "10.0.0.167", + "udp_port_imu": 7503, + "udp_port_lidar": 7502, + "udp_profile_imu": "LEGACY", + "udp_profile_lidar": "RNG19_RFL8_SIG16_NIR16" + }, + "imu_intrinsics": + { + "imu_to_sensor_transform": + [ + 1, + 0, + 0, + 6.253, + 0, + 1, + 0, + -11.775, + 0, + 0, + 1, + 7.645, + 0, + 0, + 0, + 1 + ] + }, + "lidar_data_format": + { + "column_window": + [ + 0, + 1023 + ], + "columns_per_frame": 1024, + "columns_per_packet": 16, + "pixel_shift_by_row": + [ + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12, + 12, + 4, + -4, + -12 + ], + "pixels_per_column": 128, + "udp_profile_imu": "LEGACY", + "udp_profile_lidar": "RNG19_RFL8_SIG16_NIR16" + }, + "lidar_intrinsics": + { + "lidar_to_sensor_transform": + [ + -1, + 0, + 0, + 0, + 0, + -1, + 0, + 0, + 0, + 0, + 1, + 36.18, + 0, + 0, + 0, + 1 + ] + } +} diff --git a/tests/metadata/malformed/incorrect_nbeam_angles_legacy_113.json b/tests/metadata/malformed/incorrect_nbeam_angles_legacy_113.json new file mode 100644 index 00000000..65d00656 --- /dev/null +++ b/tests/metadata/malformed/incorrect_nbeam_angles_legacy_113.json @@ -0,0 +1,18 @@ +{ + "base_pn": "000-101323-03", + "base_sn": "101847001435", + "beam_altitude_angles": [0, 0, 0, 14.799, 0, 0, 0, 12.648, 0, 0, 0, 10.537, 0, 0, 0, 8.442, 0, 0, 0, 6.357, 0, 0, 0, 4.262, 0, 0, 0, 2.183, 0, 0, 0, 0.1, 0, 0, 0, -1.991, 0, 0, 0, -4.079, 0, 0, 0, -6.168, 0, 0, 0, -8.275, 0, 0, 0, 0, 0, 0, -12.489, 0, 0, 0, -14.658, 0, 0, 0, -16.858], + "beam_azimuth_angles": [0, 0, 0, -3.387, 0, 0, 0, -3.326, 0, 0, 0, -3.288, 0, 0, 0, -3.243, 0, 0, 0, -3.206, 0, 0, 0, -3.183, 0, 0, 0, -3.159, 0, 0, 0, -3.129, 0, 0, 0, -3.1, 0, 0, 0, -3.083, 0, 0, 0, -3.067, 0, 0, 0, -3.048, 0, 0, 0, -3.04, 0, 0, 0, -3.037, 0, 0, 0, -3.043, 0, 0, 0, -3.041], + "build_date": "2019-11-04T23:58:36Z", + "build_rev": "v1.13.0", + "hostname": "os1-991937000062.local", + "image_rev": "ousteros-image-prod-aries-v1.13.0-20191105025459", + "imu_to_sensor_transform": [1, 0, 0, 6.253, 0, 1, 0, -11.775, 0, 0, 1, 7.645, 0, 0, 0, 1], + "lidar_mode": "1024x10", + "lidar_to_sensor_transform": [-1, 0, 0, 0, 0, -1, 0, 0, 0, 0, 1, 36.18, 0, 0, 0, 1], + "prod_line": "OS-1-16-A0", + "prod_pn": "840-101855-02", + "prod_sn": "991937000062", + "proto_rev": "v1.1.1", + "status": "RUNNING" +} diff --git a/tests/metadata/malformed/legacy_with_calibration_status.json b/tests/metadata/malformed/legacy_with_calibration_status.json new file mode 100644 index 00000000..f02b56b3 --- /dev/null +++ b/tests/metadata/malformed/legacy_with_calibration_status.json @@ -0,0 +1,472 @@ +{ + "base_pn": "", + "base_sn": "", + "calibration_status": + { + "reflectivity": + { + "timestamp": "2021-11-23T05:37:45", + "valid": true + } + }, + "beam_altitude_angles": + [ + 21.34, + 21.03, + 20.72, + 20.41, + 20.08, + 19.78, + 19.48, + 19.17, + 18.83, + 18.51, + 18.21, + 17.88, + 17.56, + 17.23, + 16.92, + 16.59, + 16.26, + 15.94, + 15.61, + 15.28, + 14.93, + 14.6, + 14.27, + 13.95, + 13.61, + 13.26, + 12.94, + 12.6, + 12.26, + 11.91, + 11.58, + 11.24, + 10.89, + 10.55, + 10.21, + 9.88, + 9.52, + 9.18, + 8.83, + 8.5, + 8.14, + 7.78, + 7.46, + 7.11, + 6.75, + 6.39, + 6.05, + 5.7, + 5.33, + 4.99, + 4.64, + 4.29, + 3.94, + 3.59, + 3.24, + 2.88, + 2.52, + 2.18, + 1.81, + 1.48, + 1.11, + 0.76, + 0.4, + 0.05, + -0.3, + -0.67, + -1.01, + -1.37, + -1.75, + -2.08, + -2.43, + -2.79, + -3.15, + -3.5, + -3.86, + -4.2, + -4.56, + -4.9, + -5.27, + -5.6, + -5.96, + -6.32, + -6.68, + -7, + -7.38, + -7.71, + -8.05, + -8.41, + -8.77, + -9.11, + -9.45, + -9.79, + -10.16, + -10.49, + -10.83, + -11.18, + -11.54, + -11.86, + -12.2, + -12.55, + -12.91, + -13.24, + -13.58, + -13.89, + -14.26, + -14.58, + -14.89, + -15.21, + -15.6, + -15.89, + -16.22, + -16.54, + -16.88, + -17.21, + -17.52, + -17.85, + -18.2, + -18.5, + -18.82, + -19.14, + -19.47, + -19.78, + -20.09, + -20.38, + -20.72, + -21.03, + -21.33, + -21.62 + ], + "beam_azimuth_angles": + [ + 4.27, + 1.43, + -1.39, + -4.23, + 4.26, + 1.45, + -1.38, + -4.22, + 4.28, + 1.43, + -1.39, + -4.24, + 4.27, + 1.44, + -1.4, + -4.22, + 4.28, + 1.45, + -1.39, + -4.23, + 4.26, + 1.44, + -1.4, + -4.23, + 4.27, + 1.43, + -1.39, + -4.23, + 4.26, + 1.43, + -1.4, + -4.24, + 4.26, + 1.44, + -1.4, + -4.23, + 4.28, + 1.43, + -1.4, + -4.24, + 4.26, + 1.42, + -1.39, + -4.24, + 4.27, + 1.43, + -1.42, + -4.25, + 4.26, + 1.43, + -1.42, + -4.25, + 4.28, + 1.43, + -1.42, + -4.25, + 4.26, + 1.43, + -1.44, + -4.25, + 4.27, + 1.43, + -1.43, + -4.26, + 4.27, + 1.42, + -1.43, + -4.26, + 4.25, + 1.42, + -1.43, + -4.28, + 4.25, + 1.42, + -1.45, + -4.28, + 4.26, + 1.42, + -1.44, + -4.26, + 4.26, + 1.4, + -1.46, + -4.27, + 4.24, + 1.41, + -1.43, + -4.28, + 4.26, + 1.4, + -1.44, + -4.28, + 4.22, + 1.4, + -1.43, + -4.29, + 4.25, + 1.41, + -1.45, + -4.29, + 4.24, + 1.4, + -1.46, + -4.28, + 4.24, + 1.4, + -1.44, + -4.28, + 4.22, + 1.41, + -1.45, + -4.28, + 4.25, + 1.39, + -1.45, + -4.29, + 4.23, + 1.4, + -1.45, + -4.3, + 4.22, + 1.39, + -1.47, + -4.29, + 4.22, + 1.38, + -1.47, + -4.3 + ], + "build_date": "2022-05-25T19:18:50Z", + "build_rev": "v2.3.1-rc.1", + "client_version": "ouster_client 0.4.1", + "data_format": + { + "column_window": + [ + 0, + 1023 + ], + "columns_per_frame": 1024, + "columns_per_packet": 16, + "pixel_shift_by_row": + [ + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0, + 24, + 16, + 8, + 0 + ], + "pixels_per_column": 128, + "udp_profile_imu": "LEGACY", + "udp_profile_lidar": "RNG15_RFL8_NIR8" + }, + "hostname": "", + "image_rev": "ousteros-image-dev-aries-v2.3.1-rc.1+20220525191702.patch-v2.3.x", + "imu_to_sensor_transform": + [ + 1, + 0, + 0, + 6.253, + 0, + 1, + 0, + -11.775, + 0, + 0, + 1, + 7.645, + 0, + 0, + 0, + 1 + ], + "initialization_id": 5431287, + "json_calibration_version": 4, + "lidar_mode": "1024x10", + "lidar_origin_to_beam_origin_mm": 15.806, + "lidar_to_sensor_transform": + [ + -1, + 0, + 0, + 0, + 0, + -1, + 0, + 0, + 0, + 0, + 1, + 36.18, + 0, + 0, + 0, + 1 + ], + "prod_line": "OS-1-128", + "prod_pn": "840-103575-06", + "prod_sn": "992146000760", + "proto_rev": "", + "status": "RUNNING", + "udp_port_imu": 7503, + "udp_port_lidar": 7502 +} diff --git a/tests/metadata_errors_test.cpp b/tests/metadata_errors_test.cpp new file mode 100644 index 00000000..985e6ef7 --- /dev/null +++ b/tests/metadata_errors_test.cpp @@ -0,0 +1,49 @@ +/** + * Copyright (c) 2022, Ouster, Inc. + * All rights reserved. + */ + +#include + +#include "ouster/types.h" + +class MetaErrorsFiles : public testing::TestWithParam {}; + +inline std::string getenvs(const std::string& var) { + char* res = std::getenv(var.c_str()); + return res ? std::string{res} : std::string{}; +} + +INSTANTIATE_TEST_CASE_P( + ErrorMetas, MetaErrorsFiles, + testing::Values( + "complete_but_all_zeros_legacy", // error out instead of passing to + // xyzlut + "incomplete_data_format_legacy", // missing columns_per_frame + "incomplete_data_format_nonlegacy", // missing pixels per column + "incomplete_no_sensor_info_nonlegacy", // nonlegacy can't be missing + // sensor info unlike legacy + "incomplete_no_calref_nonlegacy", // ditto calref + "garbled_legacy_and_nonlegacy", // sensor_info items on top level. + // should read as nonlegacy and fail + "legacy_with_calibration_status", // has calibration_status but nothing + // else. should read as nonlegacy and + // fail + "incorrect_nbeam_angles_legacy_113" // missing one from beam altitude + // angles + )); + +// Backwards-compatibility test for meta json parsing: compare previously parsed +// sensor_info structs to the output of metadata_from_json +TEST_P(MetaErrorsFiles, MetadataParsingExceptions) { + std::string param = GetParam(); + + auto data_dir = getenvs("DATA_DIR"); + + EXPECT_ANY_THROW({ + // parse json file + const ouster::sensor::sensor_info si = + ouster::sensor::metadata_from_json(data_dir + "/malformed/" + + param + ".json"); + }); +}