Merge Release 1.0.0 Changes to main (#235)

* squash and merge since 0.8.0 Co-authored-by: Nox-MSFT <[email protected]> Co-authored-by: Shiyi Peng <[email protected]> Co-authored-by: shiyi-peng <[email protected]>
Azure · Nov 1, 2022 · 33554d2 · 33554d2
1 parent 30126e4
commit 33554d2
Show file tree

Hide file tree

Showing 511 changed files with 30,742 additions and 11,158 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -1 +1,6 @@
 * text=auto
+
+# require LF ending for shell scripts
+*.sh text eol=lf
+
+*.png binary
diff --git a/.gitignore b/.gitignore
@@ -13,6 +13,8 @@ TAGS
 .TAGS
 !TAGS/
 tags
+tags.lock
+tags.temp
 .tags
 !tags/
 gtags.files
@@ -25,6 +27,9 @@ cscope.out
 cscope.in.out
 cscope.po.out
 
+### Python Module Files
+*__pycache__
+*__pycache__/*
 ### ADU build environment
 out/
 build/

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -35,25 +35,25 @@ include (aduc_helpers)
 # https://semver.org/spec/v2.0.0.html
 set_cache_with_env_or_default (
     ADUC_VERSION_MAJOR
-    "0"
+    "1"
     STRING
     "The major part of the semantic version")
 
 set_cache_with_env_or_default (
     ADUC_VERSION_MINOR
-    "8"
+    "0"
     STRING
     "The minor part of the semantic version")
 
 set_cache_with_env_or_default (
     ADUC_VERSION_PATCH
-    "2"
+    "0"
     STRING
     "The patch part of the semantic version")
 
 set_cache_with_env_or_default (
     ADUC_VERSION_PRERELEASE
-    "public-preview"
+    ""
     STRING
     "The pre-release part of the semantic version")
 
@@ -136,6 +136,11 @@ set (
     "content_handler.json"
     CACHE STRING "Content handler registration file name.")
 
+set (
+    ADUC_DOWNLOAD_HANDLER_REG_FILENAME
+    "download_handler.json"
+    CACHE STRING "Download handler registration file name.")
+
 set (
     ADUC_EXTENSION_REG_FILENAME
     "extension.json"
@@ -171,6 +176,16 @@ set (
     "${ADUC_EXTENSIONS_FOLDER}/${ADUC_EXTENSIONS_SUBDIR_UPDATE_CONTENT_HANDLERS}"
     CACHE STRING "Sub folder for update content handler extensions.")
 
+set (
+    ADUC_EXTENSIONS_SUBDIR_DOWNLOAD_HANDLERS
+    "download_handlers"
+    CACHE STRING "Sub folder for download handler extensions.")
+
+set (
+    ADUC_DOWNLOAD_HANDLER_EXTENSION_DIR
+    "${ADUC_EXTENSIONS_FOLDER}/${ADUC_EXTENSIONS_SUBDIR_DOWNLOAD_HANDLERS}"
+    CACHE STRING "Path for the download handler extensions")
+
 set (
     ADUC_DOWNLOADS_FOLDER
     "${ADUC_DATA_FOLDER}/downloads"
@@ -181,6 +196,20 @@ set (
     "microsoft/swupdate"
     CACHE STRING "The list of content handlers.")
 
+set (
+    ADUC_COMMANDS_FIFO_NAME
+    "${ADUC_DATA_FOLDER}/du-commands.fifo"
+    CACHE STRING "The named-pipe for commands IPC.")
+
+#
+# Starting from version 0.8.1, Device Update Agent must support only one version of the update manifest
+# based on the base dtmi model that the Agent announces when connecting to the IoT Hub.
+#
+# For this version, "dtmi:azure:iot:deviceUpdateModel;2" is using the manifest version verison 5.
+#
+set (SUPPORTED_UPDATE_MANIFEST_VERSION_MIN 4)
+set (SUPPORTED_UPDATE_MANIFEST_VERSION_MAX 5)
+
 #
 # DeviceInfo configuration
 # These values must be modified to describe your device.
@@ -324,12 +353,116 @@ set (
     "syslog"
     CACHE STRING "The syslog group.")
 
+set (
+    ADUC_IOT_HUB_PROTOCOL
+    "IotHub_Protocol_from_Config"
+    CACHE
+        STRING
+        "The protocol for Azure IotHub SDK communication. Options are MQTT, MQTT_over_WebSockets, and IotHub_Protocol_from_Config"
+)
+
+# Device Update Repo Locations
+
+set (
+    ADUC_SCRIPTS_FOLDER
+    "/scripts/"
+    CACHE STRING "The folder that holds all of the scripts")
+
+set (
+    ADUC_SCRIPTS_FOLDER_PATH
+    "${PROJECT_SOURCE_DIR}/${ADUC_SCRIPTS_FOLDER}"
+    CACHE STRING "Path to the scripts folder within the Device Update project")
+
+# Error Code Generator Script Name
+set (
+    ADUC_ERROR_CODE_GENERATOR_SCRIPT_FILENAME
+    "generate_error_code_defs.sh"
+    CACHE STRING "The script file that generates the error code definitions")
+
+# Error Code Definition Folder and Filename
+set (
+    ADUC_ERROR_CODE_DEFS_FOLDER
+    "/src/inc/aduc/"
+    CACHE STRING "The folder within Device Update to generate the result.h file to")
+
+set (
+    ADUC_GENERATED_ERROR_CODE_FOLDER_PATH
+    "${PROJECT_SOURCE_DIR}/${ADUC_ERROR_CODE_DEFS_FOLDER}"
+    CACHE STRING "Path to the folder to generate the error code file")
+
+set (
+    ADUC_ERROR_CODE_DEFS_FILENAME
+    "result.h"
+    CACHE STRING "File name for the error code definitions")
+
+# Error Code Json Folder, Filename, and Path
+set (
+    ADUC_ERROR_CODE_GENERATOR_JSON_FOLDER
+    "error_code_generator_defs"
+    CACHE STRING "Name of the folder that contains the error codes JSON file")
+
+set (
+    ADUC_ERROR_CODE_JSON_FILENAME
+    "result_codes.json"
+    CACHE STRING "Name of the json file that defines the error codes for Device Update")
+
+set (
+    ADUC_ERROR_CODE_JSON_FOLDER_PATH
+    "${ADUC_SCRIPTS_FOLDER_PATH}/${ADUC_ERROR_CODE_GENERATOR_JSON_FOLDER}"
+    CACHE STRING "Path to the folder where the JSON error code definition file lives")
+
+# Paths for the Folders
+
+get_filename_component (
+    ADUC_PATH_TO_ERROR_CODE_GENERATOR_SCRIPT
+    "${ADUC_SCRIPTS_FOLDER_PATH}/${ADUC_ERROR_CODE_GENERATOR_SCRIPT_FILENAME}"
+    ABSOLUTE
+    "/")
+
+get_filename_component (
+    ADUC_PATH_TO_ERROR_CODE_JSON_FILE
+    "${ADUC_ERROR_CODE_JSON_FOLDER_PATH}/${ADUC_ERROR_CODE_JSON_FILENAME}"
+    ABSOLUTE
+    "/")
+
+get_filename_component (
+    ADUC_GENERATED_ERROR_CODE_FILE_PATH
+    "${ADUC_GENERATED_ERROR_CODE_FOLDER_PATH}/${ADUC_ERROR_CODE_DEFS_FILENAME}"
+    ABSOLUTE
+    "/")
+
+#######
+# BEGIN Delta Downloader Handler Source Update Cache Configurations
+
+# Source Update Cache Commit Strategy
+#
+# Value values are:
+#   "TWO_PHASE_COMMIT" - Moves the incoming files then deletes previous.
+#   "OVERWRITE" - Saves space by deleting existing source updates first.
+set (
+    ADUC_DELTA_DOWNLOAD_HANDLER_SOURCE_UPDATE_CACHE_COMMIT_STRATEGY
+    "TWO_PHASE_COMMIT"
+    CACHE STRING "The commit strategy for the source update cache.")
+
+# Source Update Cache Directory
+#
+# Directory where an update from download work folder is cached after successful workflow processing
+set (
+    ADUC_DELTA_DOWNLOAD_HANDLER_SOURCE_UPDATE_CACHE_DIR
+    "${ADUC_DATA_FOLDER}/sdc"
+    CACHE STRING
+          "The base directory to cache source update payloads that delta updates are based upon.")
+
+# END Delta Downloader Handler Source Update Cache Configurations
+#######
+
 option (ADUC_WARNINGS_AS_ERRORS "Treat warnings as errors (-Werror)" ON)
 option (ADUC_BUILD_UNIT_TESTS "Build unit tests and mock some functionality" OFF)
 option (ADUC_BUILD_DOCUMENTATION "Build documentation files" OFF)
 option (ADUC_BUILD_PACKAGES "Build the ADU Agent packages" OFF)
 option (ADUC_INSTALL_DAEMON "Install the ADU Agent as a daemon" ON)
 option (ADUC_REGISTER_DAEMON "Register the ADU Agent daemon with the system" ON)
+option (ADUC_TRACE_TARGET_DEPS "Trace target dependencies" OFF)
 
 ### End CMake Options
 
@@ -350,19 +483,38 @@ get_filename_component (
     "${ADUC_CONF_FOLDER}/${DIAGNOSTICS_CONFIG_FILE}"
     ABSOLUTE
     "/")
+
 set (
     DIAGNOSTICS_CONFIG_FILE_PATH
     "${DIAGNOSTICS_CONFIG_FILE_PATH}"
     CACHE STRING "Path to the diagnostics configuration file.")
 
+# Generate the error code definition file
+execute_process (
+    COMMAND
+        ${ADUC_PATH_TO_ERROR_CODE_GENERATOR_SCRIPT} --json-file-path
+        "${ADUC_PATH_TO_ERROR_CODE_JSON_FILE}" --result-file-path
+        "${ADUC_GENERATED_ERROR_CODE_FILE_PATH}" -s
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    RESULT_VARIABLE GENERATE_ERROR_CODE_DEFS_RESULT)
+
+if (NOT
+    GENERATE_ERROR_CODE_DEFS_RESULT
+    EQUAL
+    "0")
+    message (
+        FATAL_ERROR
+            "Failed to generate error code definition from json file ${ADUC_PATH_TO_ERROR_CODE_JSON_FILE} to ${ADUC_GENERATED_ERROR_CODE_FILE_PATH} with error code ${GENERATE_ERROR_CODE_DEFS_RESULT}"
+    )
+endif ()
+
 if (ADUC_BUILD_UNIT_TESTS)
     # Need to be in the root directory to place CTestTestfile.cmake in root
     # of output folder.
     enable_testing ()
 
     copy_test_data ("${CMAKE_SOURCE_DIR}" "${ADUC_TEST_DATA_PATH_SEGMENT}"
                     "${ADUC_TEST_DATA_FOLDER}")
-
 endif ()
 
 if (ADUC_INSTALL_DAEMON)

diff --git a/SUPPORT.md b/SUPPORT.md
@@ -1,18 +1,18 @@
 # Support
 
-## How to file issues and get help
+## How to file issues and get help  
 
-* Have a technical question?
+* Have a technical question?  
 Ask on Stack Overflow with tag "iot-hub-device-update".
 
-* Found a bug, or want to request a new feature?
+* Found a bug, or want to request a new feature?  
 You can use [GitHub Issues](https://github.com/Azure/iot-hub-device-update/issues) to track bugs and feature requests. If you feel like taking a stab at fixing the bug/adding the feature, please see the [Contribution guidance here](/CONTRIBUTING.md).
 
-* Need support?
+* Need support?  
 Please document your question on [GitHub Issues](https://github.com/Azure/iot-hub-device-update/issues) thoroughly, attach logs and give repro steps, so that we can help you unblock as soon as possible. Note that currently Device Update for Iot Hub is in public preview stage, so there is no guaranteed response time, but we will get back to you as soon as we can.
 
 * You can also contact Microsoft Support, see details [here](https://docs.microsoft.com/azure/iot-hub-device-update/troubleshoot-device-update#contact)
 
-## Microsoft Support Policy
+## Microsoft Support Policy  
 
 Support for IoT Hub Device Update is limited to the resources listed above.
diff --git a/azurepipelines/build/README.md b/azurepipelines/build/README.md
@@ -0,0 +1,39 @@
+# Azure Build Pipelines ReadMe
+
+## Introduction
+
+This document is intended as a guideline to understand the structure of the current build pipeline and to add more pipelines in the future when needed.  
+
+## Build Templates
+
+Underneath the `templates/` directory there are two `*.yml` files.  
+One is `adu-native-build-steps.yml`, because the default Azure VM is running on Ubuntu 18.04 amd64 environment, this is the current supported native build. The other one is `adu-docker-build.steps.yml`, which is a template of docker build.  
+For the templates, there are two parameters: `targetOs` and `targetArch`. These parameters will be passed from build pipeline yaml files.
+
+## Native Builds
+
+The native build calls `scripts/install-deps.sh` and then calls `scripts/build.sh`.
+
+## Docker Builds
+
+The docker build pulls a container with the dependencies pre-installed from Azure Container Registry. This way, the build time can be reduced because the dependencies don't need to be installed on each build.  
+
+## Add a New Pipeline
+
+If you need to add support for a new platform, likely it does not have native Azure VM support (currently it only supports Ubuntu AMD64). In this case, you need to:
+
+1. Create a docker container image with the desirable OS and architecture, clone the ADU repo
+2. run `scripts/install-deps.sh` to install dependencies on this container
+3. Push the image to Azure Container Registry
+4. If the combination of distro & architecture already exsits in the below matrix, you would find the yaml file, and then add a `job` on that yaml file. Example: Job `BuildAduAgent_ubuntu2004` and job `BuildAduAgent_ubuntu1804` on `docker/adu-ubuntu-arm64-build.yml`.
+
+Below is a matrix of currently supported pipeline builds
+| Distro |  Architecture| Yaml File |
+|--------|--------------|-----------|
+| Ubuntu 18.04 | amd64 | native/adu-ubuntu-amd64-build.yml |
+| Ubuntu 20.04 | amd64 | docker/adu-ubuntu-amd64-build.yml |
+| Ubuntu 18.04 | arm64 | docker/adu-ubuntu-arm64-build.yml |
+| Ubuntu 20.04 | arm64 | docker/adu-ubuntu-arm64-build.yml |
+| Debian 9 | arm32 | docker/adu-debian-arm32-build.yml |
+| Debian 10 | arm64 | docker/adu-debian-arm64-build.yml |
+| Debian 10 | amd64 | docker/adu-debian-amd64-build.yml |
diff --git a/azurepipelines/build/docker/adu-debian-amd64-build.yml b/azurepipelines/build/docker/adu-debian-amd64-build.yml
@@ -1,8 +1,8 @@
 variables:
-  version.major: 0
-  version.minor: 8
+  version.major: 1
+  version.minor: 0
   version.patch: 0
-  version.pre-release: "public-preview"
+  version.pre-release: ""
   version.build: $[format('{0:yyyyMMdd}-{0:HHmmss}', pipeline.startTime)]
   # Environment variables for all client builds:
   ADUC_VERSION_MAJOR: $(version.major)

diff --git a/azurepipelines/build/docker/adu-debian-arm32-build.yml b/azurepipelines/build/docker/adu-debian-arm32-build.yml
@@ -1,8 +1,8 @@
 variables:
-  version.major: 0
-  version.minor: 8
-  version.patch: 2
-  version.pre-release: "public-preview"
+  version.major: 1
+  version.minor: 0
+  version.patch: 0
+  version.pre-release: ""
   version.build: $[format('{0:yyyyMMdd}-{0:HHmmss}', pipeline.startTime)]
 
   # Environment variables for all client builds:
@@ -44,7 +44,7 @@ pr:
   branches:
     include:
       - main
-      # - release/*
+      - release/*
       # - feature/*
       #- dev/*
   paths:
@@ -60,13 +60,13 @@ pr:
       - licenses/*
 
 jobs:
-  - job: BuildAduAgent
-    displayName: "Build ADU Agent"
+  - job: BuildAduAgent_debian10
+    displayName: "Build ADU Agent - Debian 10"
     timeoutInMinutes: 360
     cancelTimeoutInMinutes: 360
     pool: aduc_1es_client_pool
     steps:
       - template: ../templates/adu-docker-build-steps.yml
         parameters:
-          targetOs: 'debian9'
+          targetOs: 'debian10'
           targetArch: 'arm32'
diff --git a/azurepipelines/build/docker/adu-debian-arm64-build.yml b/azurepipelines/build/docker/adu-debian-arm64-build.yml
@@ -1,8 +1,8 @@
 variables:
-  version.major: 0
-  version.minor: 8
+  version.major: 1
+  version.minor: 0
   version.patch: 0
-  version.pre-release: "public-preview"
+  version.pre-release: ""
   version.build: $[format('{0:yyyyMMdd}-{0:HHmmss}', pipeline.startTime)]
   # Environment variables for all client builds:
   ADUC_VERSION_MAJOR: $(version.major)