diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index ab682adb..cb7bb086 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -46,8 +46,8 @@ jobs: echo "DFX_NO_WALLET=--no-wallet" >> "$GITHUB_ENV" fi - - name: 'Run tests' - run: bats e2e/bash/icx-asset.bash e2e/bash/icx-asset-upload.bash + # - name: 'Run tests' + # run: bats e2e/bash/icx.bash aggregate: name: e2e:required diff --git a/.github/workflows/provision-darwin.sh b/.github/workflows/provision-darwin.sh index 6b2c536e..8bc94f4d 100755 --- a/.github/workflows/provision-darwin.sh +++ b/.github/workflows/provision-darwin.sh @@ -36,11 +36,6 @@ echo "BATS_SUPPORT=${BATS_SUPPORT}" >> "$GITHUB_ENV" # Exit temporary directory. popd -# Build icx-asset -cargo build -p icx-asset -ICX_ASSET="$(pwd)/target/debug/icx-asset" -echo "ICX_ASSET=$ICX_ASSET" >> "$GITHUB_ENV" - # Build icx cargo build -p icx ICX="$(pwd)/target/debug/icx" diff --git a/.github/workflows/provision-linux.sh b/.github/workflows/provision-linux.sh index ca73a08d..73d8996a 100755 --- a/.github/workflows/provision-linux.sh +++ b/.github/workflows/provision-linux.sh @@ -34,11 +34,6 @@ echo "$HOME/bin" >> "$GITHUB_PATH" # Exit temporary directory. popd -# Build icx-asset -cargo build -p icx-asset -ICX_ASSET="$(pwd)/target/debug/icx-asset" -echo "ICX_ASSET=$ICX_ASSET" >> "$GITHUB_ENV" - # Build icx cargo build -p icx ICX="$(pwd)/target/debug/icx" diff --git a/Cargo.lock b/Cargo.lock index 926c3afc..e6fcfdaa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,12 +17,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "adler32" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" - [[package]] name = "aho-corasick" version = "0.7.18" @@ -192,15 +186,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "bstr" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" -dependencies = [ - "memchr", -] - [[package]] name = "bumpalo" version = "3.9.1" @@ -417,12 +402,6 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" -[[package]] -name = "delay" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8546bb2c80129c9c85cd2b44e160b917f34a8b7ce9f3af675502cf9960e33d62" - [[package]] name = "der" version = "0.6.0" @@ -434,17 +413,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "derivative" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "diff" version = "0.1.12" @@ -565,18 +533,6 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "279fb028e20b3c4c320317955b77c5e0c9701f05a1d309905d6fc702cdc5053e" -[[package]] -name = "flate2" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39522e96686d38f4bc984b9198e3a0613264abaebaff2c5c918bfa6b6da09af" -dependencies = [ - "cfg-if", - "crc32fast", - "libc", - "miniz_oxide", -] - [[package]] name = "fnv" version = "1.0.7" @@ -608,21 +564,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "futures" -version = "0.3.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f73fe65f54d1e12b726f517d3e2135ca3125a437b6d998caf1962961f7172d9e" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - [[package]] name = "futures-channel" version = "0.3.21" @@ -630,7 +571,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3083ce4b914124575708913bca19bfe887522d6e2e6d0952943f5eac4a74010" dependencies = [ "futures-core", - "futures-sink", ] [[package]] @@ -639,28 +579,6 @@ version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3" -[[package]] -name = "futures-executor" -version = "0.3.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9420b90cfa29e327d0429f19be13e7ddb68fa1cccb09d65e5706b8c7a749b8a6" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-intrusive" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62007592ac46aa7c2b6416f7deb9a8a8f63a01e0f1d6e1787d5630170db2b63e" -dependencies = [ - "futures-core", - "lock_api", - "parking_lot 0.11.2", -] - [[package]] name = "futures-io" version = "0.3.21" @@ -696,11 +614,9 @@ version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a" dependencies = [ - "futures-channel", "futures-core", "futures-io", "futures-macro", - "futures-sink", "futures-task", "memchr", "pin-project-lite", @@ -744,19 +660,6 @@ version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4" -[[package]] -name = "globset" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a1e17342619edbc21a964c2afbeb6c820c6a2560032872f397bb97ea127bd0a" -dependencies = [ - "aho-corasick", - "bstr", - "fnv", - "log", - "regex", -] - [[package]] name = "group" version = "0.12.0" @@ -819,9 +722,6 @@ name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -dependencies = [ - "serde", -] [[package]] name = "hmac" @@ -966,34 +866,6 @@ dependencies = [ "url", ] -[[package]] -name = "ic-asset" -version = "0.20.0" -dependencies = [ - "anyhow", - "candid", - "derivative", - "flate2", - "futures", - "futures-intrusive", - "garcon", - "globset", - "hex", - "ic-agent", - "ic-utils", - "mime", - "mime_guess", - "mockito", - "pathdiff", - "proptest", - "serde", - "serde_bytes", - "serde_json", - "sha2", - "tempfile", - "walkdir", -] - [[package]] name = "ic-identity-hsm" version = "0.20.0" @@ -1064,31 +936,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "icx-asset" -version = "0.20.0" -dependencies = [ - "anyhow", - "candid", - "chrono", - "clap", - "delay", - "garcon", - "humantime", - "ic-agent", - "ic-asset", - "ic-utils", - "libflate", - "num-traits", - "pem", - "serde", - "serde_bytes", - "serde_json", - "thiserror", - "tokio", - "walkdir", -] - [[package]] name = "icx-cert" version = "0.20.0" @@ -1229,26 +1076,6 @@ version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" -[[package]] -name = "libflate" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05605ab2bce11bcfc0e9c635ff29ef8b2ea83f29be257ee7d730cac3ee373093" -dependencies = [ - "adler32", - "crc32fast", - "libflate_lz77", -] - -[[package]] -name = "libflate_lz77" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39a734c0493409afcd49deee13c006a04e3586b9761a03543c6272c9c51f2f5a" -dependencies = [ - "rle-decode-fast", -] - [[package]] name = "libloading" version = "0.5.2" @@ -1320,16 +1147,6 @@ version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" -[[package]] -name = "mime_guess" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" -dependencies = [ - "mime", - "unicase", -] - [[package]] name = "miniz_oxide" version = "0.5.1" @@ -1546,17 +1363,6 @@ version = "6.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "029d8d0b2f198229de29dca79676f2738ff952edf3fde542eb8bf94d8c21b435" -[[package]] -name = "parking_lot" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" -dependencies = [ - "instant", - "lock_api", - "parking_lot_core 0.8.5", -] - [[package]] name = "parking_lot" version = "0.12.0" @@ -1564,21 +1370,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" dependencies = [ "lock_api", - "parking_lot_core 0.9.3", -] - -[[package]] -name = "parking_lot_core" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" -dependencies = [ - "cfg-if", - "instant", - "libc", - "redox_syscall", - "smallvec", - "winapi", + "parking_lot_core", ] [[package]] @@ -1600,12 +1392,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc" -[[package]] -name = "pathdiff" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" - [[package]] name = "pem" version = "1.0.2" @@ -1977,12 +1763,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "rle-decode-fast" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" - [[package]] name = "rustc-demangle" version = "0.1.21" @@ -2055,15 +1835,6 @@ version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - [[package]] name = "schannel" version = "0.1.20" @@ -2295,7 +2066,7 @@ checksum = "213494b7a2b503146286049378ce02b482200519accc31872ee8be91fa820a08" dependencies = [ "new_debug_unreachable", "once_cell", - "parking_lot 0.12.0", + "parking_lot", "phf_shared", "precomputed-hash", ] @@ -2468,7 +2239,7 @@ dependencies = [ "mio", "num_cpus", "once_cell", - "parking_lot 0.12.0", + "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", @@ -2601,15 +2372,6 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" -[[package]] -name = "unicase" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" -dependencies = [ - "version_check", -] - [[package]] name = "unicode-bidi" version = "0.3.8" @@ -2688,17 +2450,6 @@ dependencies = [ "libc", ] -[[package]] -name = "walkdir" -version = "2.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" -dependencies = [ - "same-file", - "winapi", - "winapi-util", -] - [[package]] name = "want" version = "0.3.0" diff --git a/Cargo.toml b/Cargo.toml index e361d367..a0a04ba5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,11 +1,9 @@ [workspace] members = [ "ic-agent", - "ic-asset", "icx-cert", "ic-identity-hsm", "ic-utils", "icx", - "icx-asset", "ref-tests" ] diff --git a/e2e/bash/icx-asset-upload.bash b/e2e/bash/icx-asset-upload.bash deleted file mode 100644 index 48e0453d..00000000 --- a/e2e/bash/icx-asset-upload.bash +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env bats - -# shellcheck disable=SC1090 -source "$BATS_SUPPORT"/load.bash - -load util/assertions - -setup() { - # We want to work from a different temporary directory for every test. - x=$(mktemp -d -t icx-asset-e2e-XXXXXXXX) - export DFX_CONFIG_ROOT="$x" - cd "$DFX_CONFIG_ROOT" || exit - - dfx new --no-frontend e2e_project - cd e2e_project || exit 1 - dfx start --background - dfx deploy -} - -teardown() { - dfx stop - rm -rf "$DFX_CONFIG_ROOT" -} - -icx_asset_sync() { - CANISTER_ID=$(dfx canister id e2e_project_assets) - assert_command "$ICX_ASSET" --pem "$DFX_CONFIG_ROOT"/.config/dfx/identity/default/identity.pem sync "$CANISTER_ID" src/e2e_project_assets/assets -} - -icx_asset_list() { - CANISTER_ID=$(dfx canister id e2e_project_assets) - assert_command "$ICX_ASSET" --pem "$DFX_CONFIG_ROOT"/.config/dfx/identity/default/identity.pem ls "$CANISTER_ID" -} - -icx_asset_upload() { - # for some reason, if you pass more than 1 parameter, and replace "$1" with "$@", - # this function doesn't call icx-asset at all. - CANISTER_ID=$(dfx canister id e2e_project_assets) - assert_command "$ICX_ASSET" --pem "$DFX_CONFIG_ROOT"/.config/dfx/identity/default/identity.pem upload "$CANISTER_ID" "$1" -} - -@test "does not delete files that are not being uploaded" { - mkdir some_dir - echo "some stuff" >some_dir/a.txt - echo "more things" >some_dir/b.txt - - icx_asset_upload /=some_dir - - icx_asset_list - - assert_match " /a.txt.*text/plain.*identity" - assert_match " /b.txt.*text/plain.*identity" - - echo "ccc" >c.txt - icx_asset_upload c.txt - - icx_asset_list - - assert_match " /a.txt.*text/plain.*identity" - assert_match " /b.txt.*text/plain.*identity" - assert_match " /c.txt.*text/plain.*identity" -} - -@test "deletes asset if necessary in order to change content type" { - # shellcheck disable=SC2086 - assert_command dfx canister ${DFX_NO_WALLET:-} call --update e2e_project_assets store '(record{key="/sample-asset.txt"; content_type="application/pdf"; content_encoding="identity"; content=blob "whatever contents!"})' - # shellcheck disable=SC2086 - assert_command dfx canister ${DFX_NO_WALLET:-} call --update e2e_project_assets store '(record{key="/sample-asset.txt"; content_type="application/pdf"; content_encoding="arbitrary"; content=blob "other contents"})' - - icx_asset_list - - assert_match " /sample-asset.txt.*application/pdf.*identity" - assert_match " /sample-asset.txt.*application/pdf.*arbitrary" - - echo "just some text" >sample-asset.txt - - # icx-asset upload should delete the asset (and upload its replacement) since the content type is different. - icx_asset_upload sample-asset.txt - - icx_asset_list - - assert_match " /sample-asset.txt.*text/plain.*identity" - assert_not_match " /sample-asset.txt.*application/pdf.*arbitrary" -} - -@test "uploads multiple files" { - echo "this is the file content" >uploaded.txt - echo "this is the file content ttt" >xyz.txt - mkdir some_dir - echo "some stuff" >some_dir/a.txt - echo "more things" >some_dir/b.txt - - CANISTER_ID=$(dfx canister id e2e_project_assets) - assert_command "$ICX_ASSET" --pem "$DFX_CONFIG_ROOT"/.config/dfx/identity/default/identity.pem upload "$CANISTER_ID" some_dir/*.txt - - icx_asset_list - - # expect: (is this surprising?) - # /a.txt - # /b.txt - - assert_match " /a.txt.*text/plain.*identity" - assert_match " /b.txt.*text/plain.*identity" -} - - -@test "uploads multiple files from absolute path" { - mkdir some_dir - echo "some stuff" >some_dir/a.txt - echo "more things" >some_dir/b.txt - - CANISTER_ID=$(dfx canister id e2e_project_assets) - assert_command "$ICX_ASSET" --pem "$DFX_CONFIG_ROOT"/.config/dfx/identity/default/identity.pem upload \ - "$CANISTER_ID" \ - "$(realpath some_dir/a.txt)" "$(realpath some_dir/b.txt)" - - icx_asset_list - - assert_match " /a.txt.*text/plain.*identity" - assert_match " /b.txt.*text/plain.*identity" -} - -@test "uploads a file by name" { - echo "this is the file content" >uploaded.txt - - icx_asset_upload uploaded.txt - - icx_asset_list - - assert_match " /uploaded.txt.*text/plain.*identity" -} - -@test "can override asset name" { - echo "this is the file content" >uploaded.txt - - icx_asset_upload /abcd.txt=uploaded.txt - - icx_asset_list - - assert_match " /abcd.txt.*text/plain.*identity" -} - -@test "uploads a directory by name" { - mkdir some_dir - echo "some stuff" >some_dir/a.txt - echo "more things" >some_dir/b.txt - - icx_asset_upload some_dir - - icx_asset_list - - # expect: - # /some_dir/a.txt - # /some_dir/b.txt - - assert_match " /some_dir/a.txt.*text/plain.*identity" - assert_match " /some_dir/b.txt.*text/plain.*identity" -} - -@test "uploads a directory by name as root" { - mkdir some_dir - echo "some stuff" >some_dir/a.txt - echo "more things" >some_dir/b.txt - - icx_asset_upload /=some_dir - - icx_asset_list - - assert_match " /a.txt.*text/plain.*identity" - assert_match " /b.txt.*text/plain.*identity" -} - diff --git a/e2e/bash/icx-asset.bash b/e2e/bash/icx-asset.bash deleted file mode 100644 index c2c269a8..00000000 --- a/e2e/bash/icx-asset.bash +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/env bats - -# shellcheck disable=SC1090 -source "$BATS_SUPPORT"/load.bash - -load util/assertions - -setup() { - cd "$(mktemp -d -t icx-asset-e2e-XXXXXXXX)" || exit 1 - dfx new --no-frontend e2e_project - cd e2e_project || exit 1 - dfx start --background - dfx deploy -} - -teardown() { - echo teardown - dfx stop -} - -icx_asset_sync() { - CANISTER_ID=$(dfx canister id e2e_project_assets) - assert_command "$ICX_ASSET" --pem "$HOME"/.config/dfx/identity/default/identity.pem sync "$CANISTER_ID" "${@:-src/e2e_project_assets/assets}" -} - -icx_asset_list() { - CANISTER_ID=$(dfx canister id e2e_project_assets) - assert_command "$ICX_ASSET" --pem "$HOME"/.config/dfx/identity/default/identity.pem ls "$CANISTER_ID" -} - -@test "lists assets" { - for i in $(seq 1 400); do - echo "some easily duplicate text $i" >>src/e2e_project_assets/assets/notreally.js - done - icx_asset_sync - - icx_asset_list - - assert_match "sample-asset.txt.*text/plain.*identity" - assert_match "notreally.js.*application/javascript.*gzip" - assert_match "notreally.js.*application/javascript.*identity" -} - -@test "creates new files" { - echo "new file content" >src/e2e_project_assets/assets/new-asset.txt - icx_asset_sync - - # shellcheck disable=SC2086 - assert_command dfx canister ${DFX_NO_WALLET:-} call --query e2e_project_assets get '(record{key="/new-asset.txt";accept_encodings=vec{"identity"}})' -} - -@test "updates existing files" { - echo -n "an asset that will change" >src/e2e_project_assets/assets/asset-to-change.txt - assert_command dfx deploy - - # shellcheck disable=SC2086 - assert_command dfx canister ${DFX_NO_WALLET:-} call --query e2e_project_assets get '(record{key="/asset-to-change.txt";accept_encodings=vec{"identity"}})' - # shellcheck disable=SC2154 - assert_match '"an asset that will change"' "$stdout" - - echo -n "an asset that has been changed" >src/e2e_project_assets/assets/asset-to-change.txt - - icx_asset_sync - - # shellcheck disable=SC2086 - assert_command dfx canister ${DFX_NO_WALLET:-} call --query e2e_project_assets get '(record{key="/asset-to-change.txt";accept_encodings=vec{"identity"}})' - # shellcheck disable=SC2154 - assert_match '"an asset that has been changed"' "$stdout" - echo pass -} - -@test "deletes removed files" { - touch src/e2e_project_assets/assets/will-delete-this.txt - dfx deploy - - assert_command dfx canister call --query e2e_project_assets get '(record{key="/will-delete-this.txt";accept_encodings=vec{"identity"}})' - assert_command dfx canister call --query e2e_project_assets list '(record{})' - assert_match '"/will-delete-this.txt"' - - rm src/e2e_project_assets/assets/will-delete-this.txt - - icx_asset_sync - - assert_command_fail dfx canister call --query e2e_project_assets get '(record{key="/will-delete-this.txt";accept_encodings=vec{"identity"}})' - assert_command dfx canister call --query e2e_project_assets list '(record{})' - assert_not_match '"/will-delete-this.txt"' -} - -@test "unsets asset encodings that are removed from project" { - - # shellcheck disable=SC2086 - assert_command dfx canister ${DFX_NO_WALLET:-} call --update e2e_project_assets store '(record{key="/sample-asset.txt"; content_type="text/plain"; content_encoding="arbitrary"; content=blob "content encoded in another way!"})' - - # shellcheck disable=SC2086 - assert_command dfx canister ${DFX_NO_WALLET:-} call --query e2e_project_assets get '(record{key="/sample-asset.txt";accept_encodings=vec{"identity"}})' - # shellcheck disable=SC2086 - assert_command dfx canister ${DFX_NO_WALLET:-} call --query e2e_project_assets get '(record{key="/sample-asset.txt";accept_encodings=vec{"arbitrary"}})' - - icx_asset_sync - - # shellcheck disable=SC2086 - assert_command dfx canister ${DFX_NO_WALLET:-} call --query e2e_project_assets get '(record{key="/sample-asset.txt";accept_encodings=vec{"identity"}})' - # shellcheck disable=SC2086 - assert_command_fail dfx canister ${DFX_NO_WALLET:-} call --query e2e_project_assets get '(record{key="/sample-asset.txt";accept_encodings=vec{"arbitrary"}})' -} - -@test "synchronizes multiple directories" { - mkdir -p multiple/a - mkdir -p multiple/b - echo "x_contents" >multiple/a/x - echo "y_contents" >multiple/b/y - - icx_asset_sync multiple/a multiple/b - # shellcheck disable=SC2086 - assert_command dfx canister ${DFX_NO_WALLET:-} call --query e2e_project_assets get '(record{key="/x";accept_encodings=vec{"identity"}})' - assert_match "x_contents" - # shellcheck disable=SC2086 - assert_command dfx canister ${DFX_NO_WALLET:-} call --query e2e_project_assets get '(record{key="/y";accept_encodings=vec{"identity"}})' - assert_match "y_contents" -} - -@test "reports errors about assets with the same key from multiple sources" { - mkdir -p multiple/a - mkdir -p multiple/b - echo "a_duplicate_contents" >multiple/a/duplicate - echo "b_duplicate_contents" >multiple/b/duplicate - - assert_command_fail icx_asset_sync multiple/a multiple/b - assert_match "Asset with key '/duplicate' defined at .*/e2e_project/multiple/b/duplicate and .*/e2e_project/multiple/a/duplicate" -} - -@test "ignores filenames and directories starting with a dot" { - touch src/e2e_project_assets/assets/.not-seen - touch src/e2e_project_assets/assets/is-seen - - mkdir -p src/e2e_project_assets/assets/.dir-skipped - touch src/e2e_project_assets/assets/.dir-skipped/also-ignored - - mkdir -p src/e2e_project_assets/assets/dir-not-skipped - touch src/e2e_project_assets/assets/dir-not-skipped/not-ignored - - icx_asset_sync - - assert_command dfx canister call --query e2e_project_assets get '(record{key="/is-seen";accept_encodings=vec{"identity"}})' - assert_command dfx canister call --query e2e_project_assets get '(record{key="/dir-not-skipped/not-ignored";accept_encodings=vec{"identity"}})' - assert_command_fail dfx canister call --query e2e_project_assets get '(record{key="/.not-seen";accept_encodings=vec{"identity"}})' - assert_command_fail dfx canister call --query e2e_project_assets get '(record{key="/.dir-skipped/also-ignored";accept_encodings=vec{"identity"}})' - - assert_command dfx canister call --query e2e_project_assets list '(record{})' - - assert_match 'is-seen' - assert_match 'not-ignored' - - assert_not_match 'not-seen' - assert_not_match 'also-ignored' -} diff --git a/ic-asset/Cargo.toml b/ic-asset/Cargo.toml deleted file mode 100644 index 3111393e..00000000 --- a/ic-asset/Cargo.toml +++ /dev/null @@ -1,47 +0,0 @@ -[package] -name = "ic-asset" -version = "0.20.0" -authors = ["DFINITY Stiftung "] -edition = "2021" -description = "Library for storing files in an asset canister." -homepage = "https://docs.rs/ic-asset" -documentation = "https://docs.rs/ic-asset" -repository = "https://github.com/dfinity/agent-rs" -license = "Apache-2.0" -readme = "README.md" -categories = ["api-bindings", "data-structures"] -keywords = ["internet-computer", "assets", "icp", "dfinity"] -include = ["src", "Cargo.toml", "../LICENSE", "README.md"] -rust-version = "1.60.0" - -[dependencies] -anyhow = "1.0" -candid = "0.7.15" -flate2 = "1.0.22" -futures = "0.3.21" -futures-intrusive = "0.4.0" -garcon = { version = "0.2", features = ["async"] } -globset = "0.4.9" -hex = {version = "0.4.2", features = ["serde"] } -ic-utils = { path = "../ic-utils", version = "0.20" } -mime = "0.3.16" -mime_guess = "2.0.4" -pathdiff = "0.2.1" -serde = "1.0" -serde_bytes = "0.11.2" -serde_json = "1.0.81" -sha2 = "0.10" -walkdir = "2.2.9" -derivative = "2.2.0" - -[dependencies.ic-agent] -version = "0.20" -path = "../ic-agent" -features = ["pem"] -default-features = false - -[dev-dependencies] -ic-agent = { path = "../ic-agent", version = "0.20" } -mockito = "0.31.0" -proptest = "1.0.0" -tempfile = "3.3.0" diff --git a/ic-asset/README.md b/ic-asset/README.md index e70eb445..49934f48 100644 --- a/ic-asset/README.md +++ b/ic-asset/README.md @@ -1,4 +1,3 @@ -`ic-asset` is a library for manipulating assets in an asset canister. - -## Useful links +# Notice +The `ic-asset` crate has been moved to the [sdk](https://github.com/dfinity/sdk) repo. diff --git a/ic-asset/src/asset_canister/batch.rs b/ic-asset/src/asset_canister/batch.rs deleted file mode 100644 index 4cdf7337..00000000 --- a/ic-asset/src/asset_canister/batch.rs +++ /dev/null @@ -1,89 +0,0 @@ -use crate::asset_canister::method_names::{COMMIT_BATCH, CREATE_BATCH}; -use crate::asset_canister::protocol::{ - BatchOperationKind, CommitBatchArguments, CreateBatchRequest, CreateBatchResponse, -}; -use crate::convenience::waiter_with_timeout; -use crate::params::CanisterCallParams; -use crate::retryable::retryable; -use candid::Nat; -use garcon::{Delay, Waiter}; - -pub(crate) async fn create_batch( - canister_call_params: &CanisterCallParams<'_>, -) -> anyhow::Result { - let mut waiter = Delay::builder() - .with(Delay::count_timeout(30)) - .exponential_backoff_capped( - std::time::Duration::from_secs(1), - 2.0, - std::time::Duration::from_secs(16), - ) - .build(); - waiter.start(); - - let result = loop { - let create_batch_args = CreateBatchRequest {}; - let response = canister_call_params - .canister - .update_(CREATE_BATCH) - .with_arg(&create_batch_args) - .build() - .map(|result: (CreateBatchResponse,)| (result.0.batch_id,)) - .call_and_wait(waiter_with_timeout(canister_call_params.timeout)) - .await; - match response { - Ok((batch_id,)) => break Ok(batch_id), - Err(agent_err) if !retryable(&agent_err) => { - break Err(agent_err); - } - Err(agent_err) => { - if let Err(_waiter_err) = waiter.async_wait().await { - break Err(agent_err); - } - } - }; - }?; - Ok(result) -} - -pub(crate) async fn commit_batch( - canister_call_params: &CanisterCallParams<'_>, - batch_id: &Nat, - operations: Vec, -) -> anyhow::Result<()> { - let mut waiter = Delay::builder() - .with(Delay::count_timeout(30)) - .exponential_backoff_capped( - std::time::Duration::from_secs(1), - 2.0, - std::time::Duration::from_secs(16), - ) - .build(); - waiter.start(); - - let arg = CommitBatchArguments { - batch_id, - operations, - }; - let result = loop { - match canister_call_params - .canister - .update_(COMMIT_BATCH) - .with_arg(&arg) - .build() - .call_and_wait(waiter_with_timeout(canister_call_params.timeout)) - .await - { - Ok(()) => break Ok(()), - Err(agent_err) if !retryable(&agent_err) => { - break Err(agent_err); - } - Err(agent_err) => { - if let Err(_waiter_err) = waiter.async_wait().await { - break Err(agent_err); - } - } - } - }?; - Ok(result) -} diff --git a/ic-asset/src/asset_canister/chunk.rs b/ic-asset/src/asset_canister/chunk.rs deleted file mode 100644 index 4db2652e..00000000 --- a/ic-asset/src/asset_canister/chunk.rs +++ /dev/null @@ -1,72 +0,0 @@ -use crate::asset_canister::method_names::CREATE_CHUNK; -use crate::asset_canister::protocol::{CreateChunkRequest, CreateChunkResponse}; -use crate::convenience::waiter_with_timeout; -use crate::params::CanisterCallParams; -use crate::retryable::retryable; -use crate::semaphores::Semaphores; -use candid::{Decode, Nat}; -use garcon::{Delay, Waiter}; - -pub(crate) async fn create_chunk( - canister_call_params: &CanisterCallParams<'_>, - batch_id: &Nat, - content: &[u8], - semaphores: &Semaphores, -) -> anyhow::Result { - let _chunk_releaser = semaphores.create_chunk.acquire(1).await; - let batch_id = batch_id.clone(); - let args = CreateChunkRequest { batch_id, content }; - - let mut waiter = Delay::builder() - .with(Delay::count_timeout(30)) - .exponential_backoff_capped( - std::time::Duration::from_secs(1), - 2.0, - std::time::Duration::from_secs(16), - ) - .build(); - waiter.start(); - - loop { - let builder = canister_call_params.canister.update_(CREATE_CHUNK); - let builder = builder.with_arg(&args); - let request_id_result = { - let _releaser = semaphores.create_chunk_call.acquire(1).await; - builder - .build() - .map(|result: (CreateChunkResponse,)| (result.0.chunk_id,)) - .call() - .await - }; - let wait_result = match request_id_result { - Ok(request_id) => { - let _releaser = semaphores.create_chunk_wait.acquire(1).await; - canister_call_params - .canister - .wait( - request_id, - waiter_with_timeout(canister_call_params.timeout), - false, - ) - .await - } - Err(err) => Err(err), - }; - match wait_result { - Ok(response) => { - // failure to decode the response is not retryable - break Decode!(&response, CreateChunkResponse) - .map_err(|e| anyhow::anyhow!("{}", e)) - .map(|x| x.chunk_id); - } - Err(agent_err) if !retryable(&agent_err) => { - break Err(anyhow::anyhow!("{}", agent_err)); - } - Err(agent_err) => { - if let Err(_waiter_err) = waiter.async_wait().await { - break Err(anyhow::anyhow!("{}", agent_err)); - } - } - } - } -} diff --git a/ic-asset/src/asset_canister/list.rs b/ic-asset/src/asset_canister/list.rs deleted file mode 100644 index 19efb598..00000000 --- a/ic-asset/src/asset_canister/list.rs +++ /dev/null @@ -1,22 +0,0 @@ -use crate::asset_canister::method_names::LIST; -use crate::asset_canister::protocol::{AssetDetails, ListAssetsRequest}; -use crate::params::CanisterCallParams; -use ic_utils::call::SyncCall; - -use std::collections::HashMap; - -pub(crate) async fn list_assets( - canister_call_params: &CanisterCallParams<'_>, -) -> anyhow::Result> { - let (entries,): (Vec,) = canister_call_params - .canister - .query_(LIST) - .with_arg(ListAssetsRequest {}) - .build() - .call() - .await?; - - let assets: HashMap<_, _> = entries.into_iter().map(|d| (d.key.clone(), d)).collect(); - - Ok(assets) -} diff --git a/ic-asset/src/asset_canister/method_names.rs b/ic-asset/src/asset_canister/method_names.rs deleted file mode 100644 index 5b275b96..00000000 --- a/ic-asset/src/asset_canister/method_names.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub(crate) const CREATE_BATCH: &str = "create_batch"; -pub(crate) const CREATE_CHUNK: &str = "create_chunk"; -pub(crate) const COMMIT_BATCH: &str = "commit_batch"; -pub(crate) const LIST: &str = "list"; diff --git a/ic-asset/src/asset_canister/mod.rs b/ic-asset/src/asset_canister/mod.rs deleted file mode 100644 index 51f7938b..00000000 --- a/ic-asset/src/asset_canister/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -pub(crate) mod batch; -pub(crate) mod chunk; -pub(crate) mod list; -pub(crate) mod method_names; -pub(crate) mod protocol; diff --git a/ic-asset/src/asset_canister/protocol.rs b/ic-asset/src/asset_canister/protocol.rs deleted file mode 100644 index 6461ce3d..00000000 --- a/ic-asset/src/asset_canister/protocol.rs +++ /dev/null @@ -1,138 +0,0 @@ -use crate::asset_config::HeadersConfig; -use candid::{CandidType, Nat}; -use serde::Deserialize; - -/// Create a new batch, which will expire after some time period. -/// This expiry is extended by any call to create_chunk(). -/// Also, removes any expired batches. -#[derive(CandidType, Debug)] -pub struct CreateBatchRequest {} - -/// The response to a CreateBatchRequest. -#[derive(CandidType, Debug, Deserialize)] -pub struct CreateBatchResponse { - /// The ID of the created batch. - pub batch_id: Nat, -} - -/// Upload a chunk of data that is part of an asset's content. -#[derive(CandidType, Debug, Deserialize)] -pub struct CreateChunkRequest<'a> { - /// The batch with which to associate the created chunk. - /// The chunk will be deleted if the batch expires before being committed. - pub batch_id: Nat, - - /// The data in this chunk. - #[serde(with = "serde_bytes")] - pub content: &'a [u8], -} - -/// The responst to a CreateChunkRequest. -#[derive(CandidType, Debug, Deserialize)] -pub struct CreateChunkResponse { - /// The ID of the created chunk. - pub chunk_id: Nat, -} - -/// Return a list of all assets in the canister. -#[derive(CandidType, Debug)] -pub struct ListAssetsRequest {} - -/// Information about a content encoding stored for an asset. -#[derive(CandidType, Debug, Deserialize)] -pub struct AssetEncodingDetails { - /// A content encoding, such as "gzip". - pub content_encoding: String, - - /// By convention, the sha256 of the entire asset encoding. This is calculated - /// by the asset uploader. It is not generated or validated by the canister. - pub sha256: Option>, -} - -/// Information about an asset stored in the canister. -#[derive(CandidType, Debug, Deserialize)] -pub struct AssetDetails { - /// The key identifies the asset. - pub key: String, - /// A list of the encodings stored for the asset. - pub encodings: Vec, - /// The MIME type of the asset. - pub content_type: String, -} - -/// Create a new asset. Has no effect if the asset already exists and the content type matches. -/// Traps if the asset already exists but with a different content type. -#[derive(CandidType, Debug)] -pub struct CreateAssetArguments { - /// The key identifies the asset. - pub key: String, - /// The MIME type of this asset - pub content_type: String, - /// The cache HTTP header Time To Live parameter - pub max_age: Option, - /// The HTTP headers - pub headers: Option, -} - -/// Set the data for a particular content encoding for the given asset. -#[derive(CandidType, Debug)] -pub struct SetAssetContentArguments { - /// The key identifies the asset. - pub key: String, - /// The content encoding for which this content applies - pub content_encoding: String, - /// The chunks to assign to this content - pub chunk_ids: Vec, - /// The sha256 of the entire content - pub sha256: Option>, -} - -/// Remove a specific content encoding for the asset. -#[derive(CandidType, Debug)] -pub struct UnsetAssetContentArguments { - /// The key identifies the asset. - pub key: String, - /// The content encoding to remove. - pub content_encoding: String, -} - -/// Remove the specified asset. -#[derive(CandidType, Debug)] -pub struct DeleteAssetArguments { - /// The key identifies the asset to delete. - pub key: String, -} - -/// Remove all assets, batches, and chunks, and reset the next batch and chunk IDs. -#[derive(CandidType, Debug)] -pub struct ClearArguments {} - -/// Batch operations that can be applied atomically. -#[derive(CandidType, Debug)] -#[allow(dead_code)] -pub enum BatchOperationKind { - /// Create a new asset. - CreateAsset(CreateAssetArguments), - - /// Assign content to an asset by encoding. - SetAssetContent(SetAssetContentArguments), - - /// Remove content from an asset by encoding. - UnsetAssetContent(UnsetAssetContentArguments), - - /// Remove an asset altogether. - DeleteAsset(DeleteAssetArguments), - - /// Clear all state from the asset canister. - Clear(ClearArguments), -} - -/// Apply all of the operations in the batch, and then remove the batch. -#[derive(CandidType, Debug)] -pub struct CommitBatchArguments<'a> { - /// The batch to commit. - pub batch_id: &'a Nat, - - /// The operations to apply atomically. - pub operations: Vec, -} diff --git a/ic-asset/src/asset_config.rs b/ic-asset/src/asset_config.rs deleted file mode 100644 index 64518a31..00000000 --- a/ic-asset/src/asset_config.rs +++ /dev/null @@ -1,639 +0,0 @@ -use anyhow::{bail, Context}; -use derivative::Derivative; -use globset::{Glob, GlobMatcher}; -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use std::{ - collections::HashMap, - fs, - path::{Path, PathBuf}, - sync::Arc, -}; - -pub(crate) const ASSETS_CONFIG_FILENAME: &str = ".ic-assets.json"; - -pub(crate) type HeadersConfig = HashMap; -type ConfigMap = HashMap>; - -#[derive(Deserialize, Serialize, Debug, Default, Clone, PartialEq, Eq)] -pub(crate) struct CacheConfig { - pub(crate) max_age: Option, -} - -#[derive(Derivative)] -#[derivative(Debug)] -struct AssetConfigRule { - #[derivative(Debug(format_with = "fmt_glob_field"))] - r#match: GlobMatcher, - cache: Option, - headers: Maybe, - ignore: Option, -} - -#[derive(Deserialize, Debug)] -enum Maybe { - Null, - Absent, - Value(T), -} - -fn fmt_glob_field( - field: &GlobMatcher, - formatter: &mut std::fmt::Formatter, -) -> Result<(), std::fmt::Error> { - formatter.write_str(field.glob().glob())?; - Ok(()) -} - -impl AssetConfigRule { - fn applies(&self, canonical_path: &Path) -> bool { - // TODO: better dot files/dirs handling, awaiting upstream changes: - // https://github.com/BurntSushi/ripgrep/issues/2229 - self.r#match.is_match(canonical_path) - } -} - -#[derive(Debug)] -pub(crate) struct AssetSourceDirectoryConfiguration { - config_map: ConfigMap, -} - -#[derive(Debug, Default, PartialEq, Eq, Serialize, Clone)] -pub(crate) struct AssetConfig { - pub(crate) cache: Option, - pub(crate) headers: Option, - pub(crate) ignore: Option, -} - -#[derive(Debug, Default)] -struct AssetConfigTreeNode { - pub parent: Option>, - pub rules: Vec, -} - -impl AssetSourceDirectoryConfiguration { - /// Constructs config tree for assets directory. - pub(crate) fn load(root_dir: &Path) -> anyhow::Result { - if !root_dir.has_root() { - bail!("root_dir paramenter is expected to be canonical path") - } - let mut config_map = HashMap::new(); - AssetConfigTreeNode::load(None, root_dir, &mut config_map)?; - - Ok(Self { config_map }) - } - - pub(crate) fn get_asset_config(&self, canonical_path: &Path) -> anyhow::Result { - let parent_dir = canonical_path.parent().with_context(|| { - format!( - "unable to get the parent directory for asset path: {:?}", - canonical_path - ) - })?; - Ok(self - .config_map - .get(parent_dir) - .with_context(|| { - format!( - "unable to find asset config for following path: {:?}", - parent_dir - ) - })? - .get_config(canonical_path)) - } -} - -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct InterimAssetConfigRule { - r#match: String, - cache: Option, - #[serde(default, deserialize_with = "deser_headers")] - headers: Maybe, - ignore: Option, -} - -impl Default for Maybe { - fn default() -> Self { - Self::Absent - } -} - -fn deser_headers<'de, D>(deserializer: D) -> Result, D::Error> -where - D: serde::Deserializer<'de>, -{ - match serde_json::value::Value::deserialize(deserializer)? { - Value::Object(v) => Ok(Maybe::Value( - v.into_iter() - .map(|(k, v)| (k, v.to_string().trim_matches('"').to_string())) - .collect::>(), - )), - Value::Null => Ok(Maybe::Null), - _ => Err(serde::de::Error::custom( - "wrong data format for field `headers` (only map or null are allowed)", - )), - } -} - -impl AssetConfigRule { - fn from_interim( - InterimAssetConfigRule { - r#match, - cache, - headers, - ignore, - }: InterimAssetConfigRule, - config_file_parent_dir: &Path, - ) -> anyhow::Result { - let glob = Glob::new( - config_file_parent_dir - .join(&r#match) - .to_str() - .with_context(|| { - format!( - "cannot combine {} and {} into a string (to be later used as a glob pattern)", - config_file_parent_dir.display(), - r#match - ) - })?, - ) - .with_context(|| format!("{} is not a valid glob pattern", r#match))?.compile_matcher(); - - Ok(Self { - r#match: glob, - cache, - headers, - ignore, - }) - } -} - -impl AssetConfigTreeNode { - fn load( - parent: Option>, - dir: &Path, - configs: &mut HashMap>, - ) -> anyhow::Result<()> { - let config_path = dir.join(ASSETS_CONFIG_FILENAME); - let mut rules = vec![]; - if config_path.exists() { - let content = fs::read(&config_path).with_context(|| { - format!("unable to read config file: {}", config_path.display()) - })?; - let interim_rules: Vec = serde_json::from_slice(&content) - .with_context(|| { - format!( - "malformed JSON asset config file: {}", - config_path.display() - ) - })?; - for interim_rule in interim_rules { - rules.push(AssetConfigRule::from_interim(interim_rule, dir)?); - } - } - - let parent_ref = match parent { - Some(p) if rules.is_empty() => p, - _ => Arc::new(Self { parent, rules }), - }; - - configs.insert(dir.to_path_buf(), parent_ref.clone()); - for f in std::fs::read_dir(&dir) - .with_context(|| format!("Unable to read directory {}", &dir.display()))? - .filter_map(|x| x.ok()) - .filter(|x| x.file_type().map_or_else(|_e| false, |ft| ft.is_dir())) - { - Self::load(Some(parent_ref.clone()), &f.path(), configs)?; - } - Ok(()) - } - - fn get_config(&self, canonical_path: &Path) -> AssetConfig { - let base_config = match &self.parent { - Some(parent) => parent.get_config(canonical_path), - None => AssetConfig::default(), - }; - self.rules - .iter() - .filter(|rule| rule.applies(canonical_path)) - .fold(base_config, |acc, x| acc.merge(x)) - } -} - -impl AssetConfig { - fn merge(mut self, other: &AssetConfigRule) -> Self { - if let Some(c) = &other.cache { - self.cache = Some(c.to_owned()); - }; - match (self.headers.as_mut(), &other.headers) { - (Some(sh), Maybe::Value(oh)) => sh.extend(oh.to_owned()), - (None, Maybe::Value(oh)) => self.headers = Some(oh.to_owned()), - (_, Maybe::Null) => self.headers = None, - (_, Maybe::Absent) => (), - }; - - if other.ignore.is_some() { - self.ignore = other.ignore; - } - self - } -} - -#[cfg(test)] -mod with_tempdir { - - use super::*; - use std::io::Write; - #[cfg(target_family = "unix")] - use std::os::unix::prelude::PermissionsExt; - use std::{collections::BTreeMap, fs::File}; - use tempfile::{Builder, TempDir}; - - fn create_temporary_assets_directory( - config_files: Option>, - assets_count: usize, - ) -> anyhow::Result { - let assets_dir = Builder::new().prefix("assets").rand_bytes(5).tempdir()?; - - let _subdirs = ["css", "js", "nested/deep"] - .map(|d| assets_dir.as_ref().join(d)) - .map(std::fs::create_dir_all); - - let _asset_files = [ - "index.html", - "js/index.js", - "js/index.map.js", - "css/main.css", - "css/stylish.css", - "nested/the-thing.txt", - "nested/deep/the-next-thing.toml", - ] - .iter() - .map(|path| assets_dir.path().join(path)) - .take(assets_count) - .for_each(|path| { - File::create(path).unwrap(); - }); - - let new_empty_config = |directory: &str| (directory.to_string(), "[]".to_string()); - let mut h = HashMap::from([ - new_empty_config(""), - new_empty_config("css"), - new_empty_config("js"), - new_empty_config("nested"), - new_empty_config("nested/deep"), - ]); - if let Some(cf) = config_files { - h.extend(cf); - } - h.into_iter().for_each(|(dir, content)| { - let path = assets_dir.path().join(dir).join(ASSETS_CONFIG_FILENAME); - let mut file = File::create(path).unwrap(); - write!(file, "{}", content).unwrap(); - }); - - Ok(assets_dir) - } - - #[test] - fn match_only_nested_files() -> anyhow::Result<()> { - let cfg = HashMap::from([( - "nested".to_string(), - r#"[{"match": "*", "cache": {"max_age": 333}}]"#.to_string(), - )]); - let assets_temp_dir = create_temporary_assets_directory(Some(cfg), 7).unwrap(); - let assets_dir = assets_temp_dir.path().canonicalize()?; - - let assets_config = AssetSourceDirectoryConfiguration::load(&assets_dir)?; - for f in ["nested/the-thing.txt", "nested/deep/the-next-thing.toml"] { - assert_eq!( - assets_config.get_asset_config(assets_dir.join(f).as_path())?, - AssetConfig { - cache: Some(CacheConfig { max_age: Some(333) }), - ..Default::default() - } - ); - } - for f in [ - "index.html", - "js/index.js", - "js/index.map.js", - "css/main.css", - "css/stylish.css", - ] { - assert_eq!( - assets_config.get_asset_config(assets_dir.join(f).as_path())?, - AssetConfig::default() - ); - } - - Ok(()) - } - - #[test] - fn overriding_cache_rules() -> anyhow::Result<()> { - let cfg = Some(HashMap::from([ - ( - "nested".to_string(), - r#"[{"match": "*", "cache": {"max_age": 111}}]"#.to_string(), - ), - ( - "".to_string(), - r#"[{"match": "*", "cache": {"max_age": 333}}]"#.to_string(), - ), - ])); - let assets_temp_dir = create_temporary_assets_directory(cfg, 7).unwrap(); - let assets_dir = assets_temp_dir.path().canonicalize()?; - - let assets_config = AssetSourceDirectoryConfiguration::load(&assets_dir)?; - for f in ["nested/the-thing.txt", "nested/deep/the-next-thing.toml"] { - assert_eq!( - assets_config.get_asset_config(assets_dir.join(f).as_path())?, - AssetConfig { - cache: Some(CacheConfig { max_age: Some(111) }), - ..Default::default() - } - ); - } - for f in [ - "index.html", - "js/index.js", - "js/index.map.js", - "css/main.css", - "css/stylish.css", - ] { - assert_eq!( - assets_config.get_asset_config(assets_dir.join(f).as_path())?, - AssetConfig { - cache: Some(CacheConfig { max_age: Some(333) }), - ..Default::default() - } - ); - } - - Ok(()) - } - - #[test] - fn overriding_headers() -> anyhow::Result<()> { - use serde_json::Value::*; - let cfg = Some(HashMap::from([( - "".to_string(), - r#" - [ - { - "match": "index.html", - "cache": { - "max_age": 22 - }, - "headers": { - "Content-Security-Policy": "add", - "x-frame-options": "NONE", - "x-content-type-options": "nosniff" - } - }, - { - "match": "*", - "headers": { - "Content-Security-Policy": "delete" - } - }, - { - "match": "*", - "headers": { - "Some-Other-Policy": "add" - } - }, - { - "match": "*", - "cache": { - "max_age": 88 - }, - "headers": { - "x-xss-protection": 1, - "x-frame-options": "SAMEORIGIN" - } - } - ] - "# - .to_string(), - )])); - let assets_temp_dir = create_temporary_assets_directory(cfg, 1).unwrap(); - let assets_dir = assets_temp_dir.path().canonicalize()?; - let assets_config = AssetSourceDirectoryConfiguration::load(&assets_dir)?; - let parsed_asset_config = - assets_config.get_asset_config(assets_dir.join("index.html").as_path())?; - let expected_asset_config = AssetConfig { - cache: Some(CacheConfig { max_age: Some(88) }), - headers: Some(HashMap::from([ - ("x-content-type-options".to_string(), "nosniff".to_string()), - ("x-frame-options".to_string(), "SAMEORIGIN".to_string()), - ("Some-Other-Policy".to_string(), "add".to_string()), - ("Content-Security-Policy".to_string(), "delete".to_string()), - ( - "x-xss-protection".to_string(), - Number(serde_json::Number::from(1)).to_string(), - ), - ])), - ..Default::default() - }; - - assert_eq!(parsed_asset_config.cache, expected_asset_config.cache); - assert_eq!( - parsed_asset_config - .headers - .unwrap() - .iter() - // keys are sorted - .collect::>(), - expected_asset_config - .headers - .unwrap() - .iter() - .collect::>(), - ); - - Ok(()) - } - - #[test] - fn prioritization() -> anyhow::Result<()> { - // 1. the most deeply nested config file takes precedens over the one in parent dir - // 2. order of rules withing file matters - last rule in config file takes precedens over the first one - let cfg = Some(HashMap::from([ - ( - "".to_string(), - r#"[ - {"match": "**/*", "cache": {"max_age": 999}}, - {"match": "nested/**/*", "cache": {"max_age": 900}}, - {"match": "nested/deep/*", "cache": {"max_age": 800}}, - {"match": "nested/**/*.toml","cache": {"max_age": 700}} - ]"# - .to_string(), - ), - ( - "nested".to_string(), - r#"[ - {"match": "the-thing.txt", "cache": {"max_age": 600}}, - {"match": "*.txt", "cache": {"max_age": 500}}, - {"match": "*", "cache": {"max_age": 400}} - ]"# - .to_string(), - ), - ( - "nested/deep".to_string(), - r#"[ - {"match": "**/*", "cache": {"max_age": 300}}, - {"match": "*", "cache": {"max_age": 200}}, - {"match": "*.toml", "cache": {"max_age": 100}} - ]"# - .to_string(), - ), - ])); - let assets_temp_dir = create_temporary_assets_directory(cfg, 7).unwrap(); - let assets_dir = assets_temp_dir.path().canonicalize()?; - - let assets_config = dbg!(AssetSourceDirectoryConfiguration::load(&assets_dir))?; - for f in [ - "index.html", - "js/index.js", - "js/index.map.js", - "css/main.css", - "css/stylish.css", - ] { - assert_eq!( - assets_config.get_asset_config(assets_dir.join(f).as_path())?, - AssetConfig { - cache: Some(CacheConfig { max_age: Some(999) }), - ..Default::default() - } - ); - } - - assert_eq!( - assets_config.get_asset_config(assets_dir.join("nested/the-thing.txt").as_path())?, - AssetConfig { - cache: Some(CacheConfig { max_age: Some(400) }), - ..Default::default() - }, - ); - assert_eq!( - assets_config - .get_asset_config(assets_dir.join("nested/deep/the-next-thing.toml").as_path())?, - AssetConfig { - cache: Some(CacheConfig { max_age: Some(100) }), - ..Default::default() - }, - ); - - Ok(()) - } - - #[test] - fn no_content_config_file() -> anyhow::Result<()> { - let cfg = Some(HashMap::from([ - ("".to_string(), "".to_string()), - ("css".to_string(), "".to_string()), - ("js".to_string(), "".to_string()), - ("nested".to_string(), "".to_string()), - ("nested/deep".to_string(), "".to_string()), - ])); - let assets_temp_dir = create_temporary_assets_directory(cfg, 0).unwrap(); - let assets_dir = assets_temp_dir.path().canonicalize()?; - let assets_config = AssetSourceDirectoryConfiguration::load(&assets_dir); - assert_eq!( - assets_config.err().unwrap().to_string(), - format!( - "malformed JSON asset config file: {}", - assets_dir.join(ASSETS_CONFIG_FILENAME).to_str().unwrap() - ) - ); - Ok(()) - } - - #[test] - fn invalid_json_config_file() -> anyhow::Result<()> { - let cfg = Some(HashMap::from([("".to_string(), "[[[{{{".to_string())])); - let assets_temp_dir = create_temporary_assets_directory(cfg, 0).unwrap(); - let assets_dir = assets_temp_dir.path().canonicalize()?; - let assets_config = AssetSourceDirectoryConfiguration::load(&assets_dir); - assert_eq!( - assets_config.err().unwrap().to_string(), - format!( - "malformed JSON asset config file: {}", - assets_dir.join(ASSETS_CONFIG_FILENAME).to_str().unwrap() - ) - ); - Ok(()) - } - - #[test] - fn invalid_glob_pattern() -> anyhow::Result<()> { - let cfg = Some(HashMap::from([( - "".to_string(), - r#"[ - {"match": "{{{\\\", "cache": {"max_age": 900}}, - ]"# - .to_string(), - )])); - let assets_temp_dir = create_temporary_assets_directory(cfg, 0).unwrap(); - let assets_dir = assets_temp_dir.path().canonicalize()?; - let assets_config = AssetSourceDirectoryConfiguration::load(&assets_dir); - assert_eq!( - assets_config.err().unwrap().to_string(), - format!( - "malformed JSON asset config file: {}", - assets_dir.join(ASSETS_CONFIG_FILENAME).to_str().unwrap() - ) - ); - Ok(()) - } - - #[test] - fn invalid_asset_path() -> anyhow::Result<()> { - let cfg = Some(HashMap::new()); - let assets_temp_dir = create_temporary_assets_directory(cfg, 0).unwrap(); - let assets_dir = assets_temp_dir.path().canonicalize()?; - let assets_config = AssetSourceDirectoryConfiguration::load(&assets_dir)?; - assert_eq!( - assets_config.get_asset_config(assets_dir.join("doesnt.exists").as_path())?, - AssetConfig::default() - ); - Ok(()) - } - - #[cfg(target_family = "unix")] - #[test] - fn no_read_permission() -> anyhow::Result<()> { - let cfg = Some(HashMap::from([( - "".to_string(), - r#"[ - {"match": "*", "cache": {"max_age": 20}} - ]"# - .to_string(), - )])); - let assets_temp_dir = create_temporary_assets_directory(cfg, 1).unwrap(); - let assets_dir = assets_temp_dir.path().canonicalize()?; - std::fs::set_permissions( - assets_dir.join(ASSETS_CONFIG_FILENAME).as_path(), - std::fs::Permissions::from_mode(0o000), - ) - .unwrap(); - - let assets_config = AssetSourceDirectoryConfiguration::load(&assets_dir); - assert_eq!( - assets_config.err().unwrap().to_string(), - format!( - "unable to read config file: {}", - assets_dir - .join(ASSETS_CONFIG_FILENAME) - .as_path() - .to_str() - .unwrap() - ) - ); - - Ok(()) - } -} diff --git a/ic-asset/src/content.rs b/ic-asset/src/content.rs deleted file mode 100644 index 17ec6617..00000000 --- a/ic-asset/src/content.rs +++ /dev/null @@ -1,45 +0,0 @@ -use crate::content_encoder::ContentEncoder; -use flate2::write::GzEncoder; -use flate2::Compression; -use mime::Mime; -use sha2::{Digest, Sha256}; -use std::io::Write; -use std::path::Path; - -pub(crate) struct Content { - pub data: Vec, - pub media_type: Mime, -} - -impl Content { - pub fn load(path: &Path) -> anyhow::Result { - let data = std::fs::read(path)?; - - // todo: check contents if mime_guess fails https://github.com/dfinity/sdk/issues/1594 - let media_type = mime_guess::from_path(path) - .first() - .unwrap_or(mime::APPLICATION_OCTET_STREAM); - - Ok(Content { data, media_type }) - } - - pub fn encode(&self, encoder: &ContentEncoder) -> anyhow::Result { - match encoder { - ContentEncoder::Gzip => self.to_gzip(), - } - } - - pub fn to_gzip(&self) -> anyhow::Result { - let mut e = GzEncoder::new(Vec::new(), Compression::default()); - e.write_all(&self.data)?; - let data = e.finish()?; - Ok(Content { - data, - media_type: self.media_type.clone(), - }) - } - - pub fn sha256(&self) -> Vec { - Sha256::digest(&self.data).to_vec() - } -} diff --git a/ic-asset/src/content_encoder.rs b/ic-asset/src/content_encoder.rs deleted file mode 100644 index 93abc4b5..00000000 --- a/ic-asset/src/content_encoder.rs +++ /dev/null @@ -1,11 +0,0 @@ -pub enum ContentEncoder { - Gzip, -} - -impl std::fmt::Display for ContentEncoder { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match &self { - ContentEncoder::Gzip => f.write_str("gzip"), - } - } -} diff --git a/ic-asset/src/convenience.rs b/ic-asset/src/convenience.rs deleted file mode 100644 index 419a2b17..00000000 --- a/ic-asset/src/convenience.rs +++ /dev/null @@ -1,6 +0,0 @@ -use garcon::Delay; -use std::time::Duration; - -pub(crate) fn waiter_with_timeout(duration: Duration) -> Delay { - Delay::builder().timeout(duration).build() -} diff --git a/ic-asset/src/lib.rs b/ic-asset/src/lib.rs deleted file mode 100644 index 3614e8a3..00000000 --- a/ic-asset/src/lib.rs +++ /dev/null @@ -1,47 +0,0 @@ -//! A library for manipulating assets in an asset canister. -//! -//! # Example -//! -//! ```rust,no_run -//! use ic_agent::agent::{Agent, http_transport::ReqwestHttpReplicaV2Transport}; -//! use ic_agent::identity::BasicIdentity; -//! use ic_utils::Canister; -//! use std::time::Duration; -//! # async fn not_main() -> Result<(), Box> { -//! # let replica_url = ""; -//! # let pemfile = ""; -//! # let canister_id = ""; -//! let agent = Agent::builder() -//! .with_transport(ReqwestHttpReplicaV2Transport::create(replica_url)?) -//! .with_identity(BasicIdentity::from_pem_file(pemfile)?) -//! .build()?; -//! let canister = Canister::builder() -//! .with_canister_id(canister_id) -//! .with_agent(&agent) -//! .build()?; -//! ic_asset::sync(&canister, &[concat!(env!("CARGO_MANIFEST_DIR"), "assets/").as_ref()], Duration::from_secs(60)).await?; -//! # Ok(()) -//! # } - -#![deny( - missing_docs, - missing_debug_implementations, - rustdoc::broken_intra_doc_links, - rustdoc::private_intra_doc_links -)] - -mod asset_canister; -mod asset_config; -mod content; -mod content_encoder; -mod convenience; -mod operations; -mod params; -mod plumbing; -mod retryable; -mod semaphores; -mod sync; -mod upload; - -pub use sync::sync; -pub use upload::upload; diff --git a/ic-asset/src/operations.rs b/ic-asset/src/operations.rs deleted file mode 100644 index 85931f4d..00000000 --- a/ic-asset/src/operations.rs +++ /dev/null @@ -1,123 +0,0 @@ -use crate::asset_canister::protocol::{ - AssetDetails, BatchOperationKind, CreateAssetArguments, DeleteAssetArguments, - SetAssetContentArguments, UnsetAssetContentArguments, -}; -use crate::plumbing::ProjectAsset; -use std::collections::HashMap; - -pub(crate) fn delete_obsolete_assets( - operations: &mut Vec, - project_assets: &HashMap, - container_assets: &mut HashMap, -) { - let mut deleted_container_assets = vec![]; - for (key, container_asset) in container_assets.iter() { - let project_asset = project_assets.get(key); - if project_asset - .filter(|&x| x.media_type.to_string() == container_asset.content_type) - .is_none() - { - operations.push(BatchOperationKind::DeleteAsset(DeleteAssetArguments { - key: key.clone(), - })); - deleted_container_assets.push(key.clone()); - } - } - for k in deleted_container_assets { - container_assets.remove(&k); - } -} - -pub(crate) fn delete_incompatible_assets( - operations: &mut Vec, - project_assets: &HashMap, - container_assets: &mut HashMap, -) { - let mut deleted_container_assets = vec![]; - for (key, container_asset) in container_assets.iter() { - if let Some(project_asset) = project_assets.get(key) { - if project_asset.media_type.to_string() != container_asset.content_type { - operations.push(BatchOperationKind::DeleteAsset(DeleteAssetArguments { - key: key.clone(), - })); - deleted_container_assets.push(key.clone()); - } - } - } - for k in deleted_container_assets { - container_assets.remove(&k); - } -} - -pub(crate) fn create_new_assets( - operations: &mut Vec, - project_assets: &HashMap, - container_assets: &HashMap, -) { - for (key, project_asset) in project_assets { - if !container_assets.contains_key(key) { - let max_age = project_asset - .asset_descriptor - .config - .cache - .as_ref() - .and_then(|c| c.max_age); - - let headers = project_asset.asset_descriptor.config.clone().headers; - - operations.push(BatchOperationKind::CreateAsset(CreateAssetArguments { - key: key.clone(), - content_type: project_asset.media_type.to_string(), - max_age, - headers, - })); - } - } -} - -pub(crate) fn unset_obsolete_encodings( - operations: &mut Vec, - project_assets: &HashMap, - container_assets: &HashMap, -) { - for (key, details) in container_assets { - // delete_obsolete_assets handles the case where key is not found in project_assets - if let Some(project_asset) = project_assets.get(key) { - for encoding_details in &details.encodings { - let project_contains_encoding = project_asset - .encodings - .contains_key(&encoding_details.content_encoding); - if !project_contains_encoding { - operations.push(BatchOperationKind::UnsetAssetContent( - UnsetAssetContentArguments { - key: key.clone(), - content_encoding: encoding_details.content_encoding.clone(), - }, - )); - } - } - } - } -} - -pub(crate) fn set_encodings( - operations: &mut Vec, - project_assets: HashMap, -) { - for (key, project_asset) in project_assets { - for (content_encoding, v) in project_asset.encodings { - if v.already_in_place { - continue; - } - - operations.push(BatchOperationKind::SetAssetContent( - SetAssetContentArguments { - key: key.clone(), - content_encoding, - chunk_ids: v.chunk_ids, - sha256: Some(v.sha256), - }, - )); - } - } -} diff --git a/ic-asset/src/params.rs b/ic-asset/src/params.rs deleted file mode 100644 index da393329..00000000 --- a/ic-asset/src/params.rs +++ /dev/null @@ -1,7 +0,0 @@ -use ic_utils::Canister; -use std::time::Duration; - -pub(crate) struct CanisterCallParams<'a> { - pub(crate) canister: &'a Canister<'a>, - pub(crate) timeout: Duration, -} diff --git a/ic-asset/src/plumbing.rs b/ic-asset/src/plumbing.rs deleted file mode 100644 index 3ae94d50..00000000 --- a/ic-asset/src/plumbing.rs +++ /dev/null @@ -1,309 +0,0 @@ -use crate::asset_canister::protocol::AssetDetails; -use crate::asset_config::AssetConfig; -use crate::content::Content; -use crate::content_encoder::ContentEncoder; -use crate::params::CanisterCallParams; - -use crate::asset_canister::chunk::create_chunk; -use crate::semaphores::Semaphores; -use candid::Nat; -use futures::future::try_join_all; -use futures::TryFutureExt; -use mime::Mime; -use std::collections::HashMap; -use std::path::PathBuf; - -const CONTENT_ENCODING_IDENTITY: &str = "identity"; - -// The most mb any one file is considered to have for purposes of limiting data loaded at once. -// Any file counts as at least 1 mb. -const MAX_COST_SINGLE_FILE_MB: usize = 45; - -const MAX_CHUNK_SIZE: usize = 1_900_000; - -#[derive(Clone, Debug)] -pub(crate) struct AssetDescriptor { - pub(crate) source: PathBuf, - pub(crate) key: String, - pub(crate) config: AssetConfig, -} - -pub(crate) struct ProjectAssetEncoding { - pub(crate) chunk_ids: Vec, - pub(crate) sha256: Vec, - pub(crate) already_in_place: bool, -} - -pub(crate) struct ProjectAsset { - pub(crate) asset_descriptor: AssetDescriptor, - pub(crate) media_type: Mime, - pub(crate) encodings: HashMap, -} - -#[allow(clippy::too_many_arguments)] -async fn make_project_asset_encoding( - canister_call_params: &CanisterCallParams<'_>, - batch_id: &Nat, - asset_descriptor: &AssetDescriptor, - container_assets: &HashMap, - content: &Content, - content_encoding: &str, - semaphores: &Semaphores, -) -> anyhow::Result { - let sha256 = content.sha256(); - - let already_in_place = - if let Some(container_asset) = container_assets.get(&asset_descriptor.key) { - if container_asset.content_type != content.media_type.to_string() { - false - } else if let Some(container_asset_encoding_sha256) = container_asset - .encodings - .iter() - .find(|details| details.content_encoding == content_encoding) - .and_then(|details| details.sha256.as_ref()) - { - container_asset_encoding_sha256 == &sha256 - } else { - false - } - } else { - false - }; - - let chunk_ids = if already_in_place { - println!( - " {}{} ({} bytes) sha {} is already installed", - &asset_descriptor.key, - content_encoding_descriptive_suffix(content_encoding), - content.data.len(), - hex::encode(&sha256), - ); - vec![] - } else { - upload_content_chunks( - canister_call_params, - batch_id, - asset_descriptor, - content, - content_encoding, - semaphores, - ) - .await? - }; - - Ok(ProjectAssetEncoding { - chunk_ids, - sha256, - already_in_place, - }) -} - -#[allow(clippy::too_many_arguments)] -async fn make_encoding( - canister_call_params: &CanisterCallParams<'_>, - batch_id: &Nat, - asset_descriptor: &AssetDescriptor, - container_assets: &HashMap, - content: &Content, - encoder: &Option, - semaphores: &Semaphores, -) -> anyhow::Result> { - match encoder { - None => { - let identity_asset_encoding = make_project_asset_encoding( - canister_call_params, - batch_id, - asset_descriptor, - container_assets, - content, - CONTENT_ENCODING_IDENTITY, - semaphores, - ) - .await?; - Ok(Some(( - CONTENT_ENCODING_IDENTITY.to_string(), - identity_asset_encoding, - ))) - } - Some(encoder) => { - let encoded = content.encode(encoder)?; - if encoded.data.len() < content.data.len() { - let content_encoding = format!("{}", encoder); - let project_asset_encoding = make_project_asset_encoding( - canister_call_params, - batch_id, - asset_descriptor, - container_assets, - &encoded, - &content_encoding, - semaphores, - ) - .await?; - Ok(Some((content_encoding, project_asset_encoding))) - } else { - Ok(None) - } - } - } -} - -async fn make_encodings( - canister_call_params: &CanisterCallParams<'_>, - batch_id: &Nat, - asset_descriptor: &AssetDescriptor, - container_assets: &HashMap, - content: &Content, - semaphores: &Semaphores, -) -> anyhow::Result> { - let mut encoders = vec![None]; - for encoder in applicable_encoders(&content.media_type) { - encoders.push(Some(encoder)); - } - - let encoding_futures: Vec<_> = encoders - .iter() - .map(|maybe_encoder| { - make_encoding( - canister_call_params, - batch_id, - asset_descriptor, - container_assets, - content, - maybe_encoder, - semaphores, - ) - }) - .collect(); - - let encodings = try_join_all(encoding_futures).await?; - - let mut result: HashMap = HashMap::new(); - - for (key, value) in encodings.into_iter().flatten() { - result.insert(key, value); - } - Ok(result) -} - -async fn make_project_asset( - canister_call_params: &CanisterCallParams<'_>, - batch_id: &Nat, - asset_descriptor: AssetDescriptor, - container_assets: &HashMap, - semaphores: &Semaphores, -) -> anyhow::Result { - let file_size = std::fs::metadata(&asset_descriptor.source)?.len(); - let permits = std::cmp::max( - 1, - std::cmp::min( - ((file_size + 999999) / 1000000) as usize, - MAX_COST_SINGLE_FILE_MB, - ), - ); - let _releaser = semaphores.file.acquire(permits).await; - let content = Content::load(&asset_descriptor.source)?; - - let encodings = make_encodings( - canister_call_params, - batch_id, - &asset_descriptor, - container_assets, - &content, - semaphores, - ) - .await?; - - Ok(ProjectAsset { - asset_descriptor, - media_type: content.media_type, - encodings, - }) -} - -pub(crate) async fn make_project_assets( - canister_call_params: &CanisterCallParams<'_>, - batch_id: &Nat, - asset_descriptors: Vec, - container_assets: &HashMap, -) -> anyhow::Result> { - let semaphores = Semaphores::new(); - - let project_asset_futures: Vec<_> = asset_descriptors - .iter() - .map(|loc| { - make_project_asset( - canister_call_params, - batch_id, - loc.clone(), - container_assets, - &semaphores, - ) - }) - .collect(); - let project_assets = try_join_all(project_asset_futures).await?; - - let mut hm = HashMap::new(); - for project_asset in project_assets { - hm.insert(project_asset.asset_descriptor.key.clone(), project_asset); - } - Ok(hm) -} - -async fn upload_content_chunks( - canister_call_params: &CanisterCallParams<'_>, - batch_id: &Nat, - asset_descriptor: &AssetDescriptor, - content: &Content, - content_encoding: &str, - semaphores: &Semaphores, -) -> anyhow::Result> { - if content.data.is_empty() { - let empty = vec![]; - let chunk_id = create_chunk(canister_call_params, batch_id, &empty, semaphores).await?; - println!( - " {}{} 1/1 (0 bytes)", - &asset_descriptor.key, - content_encoding_descriptive_suffix(content_encoding) - ); - return Ok(vec![chunk_id]); - } - - let count = (content.data.len() + MAX_CHUNK_SIZE - 1) / MAX_CHUNK_SIZE; - let chunks_futures: Vec<_> = content - .data - .chunks(MAX_CHUNK_SIZE) - .enumerate() - .map(|(i, data_chunk)| { - create_chunk(canister_call_params, batch_id, data_chunk, semaphores).map_ok( - move |chunk_id| { - println!( - " {}{} {}/{} ({} bytes)", - &asset_descriptor.key, - content_encoding_descriptive_suffix(content_encoding), - i + 1, - count, - data_chunk.len(), - ); - chunk_id - }, - ) - }) - .collect(); - try_join_all(chunks_futures).await -} - -fn content_encoding_descriptive_suffix(content_encoding: &str) -> String { - if content_encoding == CONTENT_ENCODING_IDENTITY { - "".to_string() - } else { - format!(" ({})", content_encoding) - } -} - -// todo: make this configurable https://github.com/dfinity/dx-triage/issues/152 -fn applicable_encoders(media_type: &Mime) -> Vec { - match (media_type.type_(), media_type.subtype()) { - (mime::TEXT, _) | (_, mime::JAVASCRIPT) | (_, mime::HTML) => vec![ContentEncoder::Gzip], - _ => vec![], - } -} diff --git a/ic-asset/src/retryable.rs b/ic-asset/src/retryable.rs deleted file mode 100644 index f661eb7e..00000000 --- a/ic-asset/src/retryable.rs +++ /dev/null @@ -1,29 +0,0 @@ -use ic_agent::agent_error::HttpErrorPayload; -use ic_agent::AgentError; - -pub(crate) fn retryable(agent_error: &AgentError) -> bool { - match agent_error { - AgentError::ReplicaError { - reject_code, - reject_message, - } if *reject_code == 5 && reject_message.contains("is out of cycles") => false, - AgentError::ReplicaError { - reject_code, - reject_message, - } if *reject_code == 5 && reject_message.contains("Fail to decode") => false, - AgentError::ReplicaError { - reject_code, - reject_message, - } if *reject_code == 4 && reject_message.contains("is not authorized") => false, - AgentError::HttpError(HttpErrorPayload { - status, - content_type: _, - content: _, - }) if *status == 403 => { - // sometimes out of cycles looks like this - // assume any 403(unauthorized) is not retryable - false - } - _ => true, - } -} diff --git a/ic-asset/src/semaphores.rs b/ic-asset/src/semaphores.rs deleted file mode 100644 index 835aedd2..00000000 --- a/ic-asset/src/semaphores.rs +++ /dev/null @@ -1,53 +0,0 @@ -use futures_intrusive::sync::SharedSemaphore; - -// Maximum MB of file data to load at once. More memory may be used, due to encodings. -const MAX_SIMULTANEOUS_LOADED_MB: usize = 50; - -// How many simultaneous chunks being created at once -const MAX_SIMULTANEOUS_CREATE_CHUNK: usize = 12; - -// How many simultaneous Agent.call() to create_chunk -const MAX_SIMULTANEOUS_CREATE_CHUNK_CALLS: usize = 4; - -// How many simultaneous Agent.wait() on create_chunk result -const MAX_SIMULTANEOUS_CREATE_CHUNK_WAITS: usize = 4; - -pub(crate) struct Semaphores { - // The "file" semaphore limits how much file data to load at once. A given loaded file's data - // may be simultaneously encoded (gzip and so forth). - pub file: SharedSemaphore, - - // The create_chunk semaphore limits the number of chunks that can be in the process - // of being created at one time. Since each chunk creation can involve retries, - // this focuses those retries on a smaller number of chunks. - // Without this semaphore, every chunk would make its first attempt, before - // any chunk made its second attempt. - pub create_chunk: SharedSemaphore, - - // The create_chunk_call semaphore limits the number of simultaneous - // agent.call()s to create_chunk. - pub create_chunk_call: SharedSemaphore, - - // The create_chunk_wait semaphore limits the number of simultaneous - // agent.wait() calls for outstanding create_chunk requests. - pub create_chunk_wait: SharedSemaphore, -} - -impl Semaphores { - pub fn new() -> Semaphores { - let file = SharedSemaphore::new(true, MAX_SIMULTANEOUS_LOADED_MB); - - let create_chunk = SharedSemaphore::new(true, MAX_SIMULTANEOUS_CREATE_CHUNK); - - let create_chunk_call = SharedSemaphore::new(true, MAX_SIMULTANEOUS_CREATE_CHUNK_CALLS); - - let create_chunk_wait = SharedSemaphore::new(true, MAX_SIMULTANEOUS_CREATE_CHUNK_WAITS); - - Semaphores { - file, - create_chunk, - create_chunk_call, - create_chunk_wait, - } - } -} diff --git a/ic-asset/src/sync.rs b/ic-asset/src/sync.rs deleted file mode 100644 index 33cdec66..00000000 --- a/ic-asset/src/sync.rs +++ /dev/null @@ -1,699 +0,0 @@ -use crate::asset_canister::batch::{commit_batch, create_batch}; -use crate::asset_canister::list::list_assets; -use crate::asset_canister::protocol::{AssetDetails, BatchOperationKind}; -use crate::asset_config::{AssetConfig, AssetSourceDirectoryConfiguration, ASSETS_CONFIG_FILENAME}; -use crate::params::CanisterCallParams; - -use crate::operations::{ - create_new_assets, delete_obsolete_assets, set_encodings, unset_obsolete_encodings, -}; -use crate::plumbing::{make_project_assets, AssetDescriptor, ProjectAsset}; -use anyhow::{bail, Context}; -use ic_utils::Canister; -use std::collections::HashMap; -use std::path::Path; -use std::time::Duration; -use walkdir::WalkDir; - -/// Sets the contents of the asset canister to the contents of a directory, including deleting old assets. -pub async fn sync( - canister: &Canister<'_>, - dirs: &[&Path], - timeout: Duration, -) -> anyhow::Result<()> { - let asset_descriptors = gather_asset_descriptors(dirs)?; - - let canister_call_params = CanisterCallParams { canister, timeout }; - - let container_assets = list_assets(&canister_call_params).await?; - - println!("Starting batch."); - - let batch_id = create_batch(&canister_call_params).await?; - - println!("Staging contents of new and changed assets:"); - - let project_assets = make_project_assets( - &canister_call_params, - &batch_id, - asset_descriptors, - &container_assets, - ) - .await?; - - let operations = assemble_synchronization_operations(project_assets, container_assets); - - println!("Committing batch."); - commit_batch(&canister_call_params, &batch_id, operations).await?; - - Ok(()) -} - -fn include_entry(entry: &walkdir::DirEntry, config: &AssetConfig) -> bool { - let starts_with_a_dot = entry - .file_name() - .to_str() - .map(|s| s.starts_with('.')) - .unwrap_or(false); - - match (starts_with_a_dot, config.ignore) { - (dot, None) => !dot, - (_dot, Some(ignored)) => !ignored, - } -} - -fn gather_asset_descriptors(dirs: &[&Path]) -> anyhow::Result> { - let mut asset_descriptors: HashMap = HashMap::new(); - for dir in dirs { - let dir = dir.canonicalize().with_context(|| { - format!( - "unable to canonicalize the following path: {}", - dir.display() - ) - })?; - let configuration = AssetSourceDirectoryConfiguration::load(&dir)?; - let mut asset_descriptors_interim = vec![]; - for e in WalkDir::new(&dir) - .into_iter() - .filter_entry(|entry| { - if let Ok(canonical_path) = &entry.path().canonicalize() { - let config = configuration - .get_asset_config(canonical_path) - .unwrap_or_default(); - include_entry(entry, &config) - } else { - false - } - }) - .filter_map(|r| r.ok()) - .filter(|entry| { - entry.file_type().is_file() && entry.file_name() != ASSETS_CONFIG_FILENAME - }) - { - let source = e.path().canonicalize().with_context(|| { - format!( - "unable to canonicalize the path when gathering asset descriptors: {}", - dir.display() - ) - })?; - let relative = source.strip_prefix(&dir).expect("cannot strip prefix"); - let key = String::from("/") + relative.to_string_lossy().as_ref(); - let config = configuration.get_asset_config(&source).context(format!( - "failed to get config for asset: {}", - source.display() - ))?; - - asset_descriptors_interim.push(AssetDescriptor { - source, - key, - config, - }) - } - - for asset_descriptor in asset_descriptors_interim { - if let Some(already_seen) = asset_descriptors.get(&asset_descriptor.key) { - bail!( - "Asset with key '{}' defined at {} and {}", - &asset_descriptor.key, - asset_descriptor.source.display(), - already_seen.source.display() - ) - } - asset_descriptors.insert(asset_descriptor.key.clone(), asset_descriptor); - } - } - Ok(asset_descriptors.into_values().collect()) -} - -fn assemble_synchronization_operations( - project_assets: HashMap, - container_assets: HashMap, -) -> Vec { - let mut container_assets = container_assets; - - let mut operations = vec![]; - - delete_obsolete_assets(&mut operations, &project_assets, &mut container_assets); - create_new_assets(&mut operations, &project_assets, &container_assets); - unset_obsolete_encodings(&mut operations, &project_assets, &container_assets); - set_encodings(&mut operations, project_assets); - - operations -} - -#[cfg(test)] -mod test_gathering_asset_descriptors_with_tempdir { - - use crate::asset_config::{CacheConfig, HeadersConfig}; - - use super::{gather_asset_descriptors, AssetDescriptor}; - use std::{ - collections::HashMap, - fs, - path::{Path, PathBuf}, - }; - use tempfile::{Builder, TempDir}; - - impl AssetDescriptor { - fn default_from_path(assets_dir: &Path, relative_path: &str) -> Self { - let relative_path = relative_path.split('/').collect::>(); - let relative_path = relative_path - .iter() - .fold(PathBuf::new(), |acc, x| acc.join(x)); - AssetDescriptor { - source: assets_dir.join(&relative_path), - key: format!("/{}", relative_path.to_str().unwrap()), - config: Default::default(), - } - } - fn with_headers(mut self, headers: HashMap<&str, &str>) -> Self { - let headers = headers - .into_iter() - .map(|(k, v)| (k.to_string(), v.to_string())) - .collect::(); - let mut h = self.config.headers.unwrap_or_default(); - h.extend(headers); - self.config.headers = Some(h); - self - } - fn with_cache(mut self, cache: CacheConfig) -> Self { - self.config.cache = Some(cache); - self - } - } - - impl PartialEq for AssetDescriptor { - fn eq(&self, other: &Self) -> bool { - [ - self.source == other.source, - self.key == other.key, - self.config.cache == other.config.cache, - self.config.headers == other.config.headers, - self.config.ignore.unwrap_or(false) == other.config.ignore.unwrap_or(false), - ] - .into_iter() - .all(|v| v) - } - } - /// assets_tempdir directory structure: - /// /assetsRAND5 - /// ├── .ic-assets.json - /// ├── .hfile - /// ├── file - /// ├─- .hidden-dir - /// │ ├── .ic-assets.json - /// │ ├── .hfile - /// │ ├── file - /// │ └── .hidden-dir-nested - /// │ ├── .ic-assets.json - /// │ ├── .hfile - /// │ └── file - /// └── .hidden-dir-flat - /// ├── .ic-assets.json - /// ├── .hfile - /// └── file - fn create_temporary_assets_directory( - modified_files: HashMap, - ) -> anyhow::Result { - let assets_tempdir = Builder::new().prefix("assets").rand_bytes(5).tempdir()?; - - let mut default_files = HashMap::from([ - (Path::new(".ic-assets.json").to_path_buf(), "[]".to_string()), - (Path::new(".hfile").to_path_buf(), "".to_string()), - (Path::new("file").to_path_buf(), "".to_string()), - ( - Path::new(".hidden-dir/.ic-assets.json").to_path_buf(), - "[]".to_string(), - ), - ( - Path::new(".hidden-dir/.hfile").to_path_buf(), - "".to_string(), - ), - (Path::new(".hidden-dir/file").to_path_buf(), "".to_string()), - ( - Path::new(".hidden-dir/.hidden-dir-nested/.ic-assets.json").to_path_buf(), - "[]".to_string(), - ), - ( - Path::new(".hidden-dir/.hidden-dir-nested/.hfile").to_path_buf(), - "".to_string(), - ), - ( - Path::new(".hidden-dir/.hidden-dir-nested/file").to_path_buf(), - "".to_string(), - ), - ( - Path::new(".hidden-dir-flat/.ic-assets.json").to_path_buf(), - "[]".to_string(), - ), - ( - Path::new(".hidden-dir-flat/.hfile").to_path_buf(), - "".to_string(), - ), - ( - Path::new(".hidden-dir-flat/file").to_path_buf(), - "".to_string(), - ), - ]); - default_files.extend(modified_files); - - for (k, v) in default_files { - let path = assets_tempdir.path().join(k); - fs::create_dir_all(path.parent().unwrap()).unwrap(); - fs::write(path, v).unwrap(); - } - - Ok(assets_tempdir) - } - - #[test] - /// test gathering all files (including dotfiles in nested dotdirs) - fn gather_all_files() { - let files = HashMap::from([( - Path::new(".ic-assets.json").to_path_buf(), - r#"[ - {"match": ".*", "ignore": false} - ]"# - .to_string(), - )]); - - let assets_temp_dir = create_temporary_assets_directory(files).unwrap(); - let assets_dir = assets_temp_dir.path().canonicalize().unwrap(); - let mut asset_descriptors = dbg!(gather_asset_descriptors(&[&assets_dir]).unwrap()); - - let mut expected_asset_descriptors = vec![ - AssetDescriptor::default_from_path(&assets_dir, ".hfile"), - AssetDescriptor::default_from_path(&assets_dir, "file"), - AssetDescriptor::default_from_path( - &assets_dir, - ".hidden-dir/.hidden-dir-nested/.hfile", - ), - AssetDescriptor::default_from_path(&assets_dir, ".hidden-dir/.hidden-dir-nested/file"), - AssetDescriptor::default_from_path(&assets_dir, ".hidden-dir-flat/.hfile"), - AssetDescriptor::default_from_path(&assets_dir, ".hidden-dir-flat/file"), - AssetDescriptor::default_from_path(&assets_dir, ".hidden-dir/.hfile"), - AssetDescriptor::default_from_path(&assets_dir, ".hidden-dir/file"), - ]; - - expected_asset_descriptors.sort_by_key(|v| v.source.clone()); - asset_descriptors.sort_by_key(|v| v.source.clone()); - assert_eq!(asset_descriptors, expected_asset_descriptors); - } - - #[test] - /// test gathering all non-dot files, from non-dot dirs - fn gather_all_nondot_files_from_nondot_dirs() { - let files = HashMap::from([( - Path::new(".ic-assets.json").to_path_buf(), - r#"[ - {"match": ".*", "ignore": true} - ]"# - .to_string(), - )]); - let assets_temp_dir = create_temporary_assets_directory(files).unwrap(); - let assets_dir = assets_temp_dir.path().canonicalize().unwrap(); - let asset_descriptors = gather_asset_descriptors(&[&assets_dir]).unwrap(); - let expected_asset_descriptors = - vec![AssetDescriptor::default_from_path(&assets_dir, "file")]; - assert_eq!(asset_descriptors, expected_asset_descriptors); - - // same but without the `ignore` flag (defaults to `true`) - let files = HashMap::from([( - Path::new(".ic-assets.json").to_path_buf(), - r#"[ - {"match": ".*"} - ]"# - .to_string(), - )]); - let assets_temp_dir = create_temporary_assets_directory(files).unwrap(); - let assets_dir = assets_temp_dir.path().canonicalize().unwrap(); - let asset_descriptors = gather_asset_descriptors(&[&assets_dir]).unwrap(); - let expected_asset_descriptors = - vec![AssetDescriptor::default_from_path(&assets_dir, "file")]; - assert_eq!(asset_descriptors, expected_asset_descriptors); - - // different glob pattern - let files = HashMap::from([( - Path::new(".ic-assets.json").to_path_buf(), - r#"[ - {"match": "*"} - ]"# - .to_string(), - )]); - let assets_temp_dir = create_temporary_assets_directory(files).unwrap(); - let assets_dir = assets_temp_dir.path().canonicalize().unwrap(); - let asset_descriptors = gather_asset_descriptors(&[&assets_dir]).unwrap(); - let expected_asset_descriptors = - vec![AssetDescriptor::default_from_path(&assets_dir, "file")]; - assert_eq!(asset_descriptors, expected_asset_descriptors); - - // different glob pattern - let files = HashMap::from([( - Path::new(".ic-assets.json").to_path_buf(), - r#"[ - {"match": "**/*"} - ]"# - .to_string(), - )]); - let assets_temp_dir = create_temporary_assets_directory(files).unwrap(); - let assets_dir = assets_temp_dir.path().canonicalize().unwrap(); - let asset_descriptors = gather_asset_descriptors(&[&assets_dir]).unwrap(); - let expected_asset_descriptors = - vec![AssetDescriptor::default_from_path(&assets_dir, "file")]; - assert_eq!(asset_descriptors, expected_asset_descriptors); - } - - #[cfg(target_family = "unix")] - #[test] - /// Cannot include files inside hidden directory using only config file - /// inside hidden directory. Hidden directory has to be first included in - /// config file sitting in parent dir. - /// The behaviour will have to stay until this lands: - /// https://github.com/BurntSushi/ripgrep/issues/2229 - fn failed_to_include_hidden_dir() { - let files = HashMap::from([( - Path::new(".hidden-dir/.ic-assets.json").to_path_buf(), - r#"[ - {"match": ".", "ignore": false}, - {"match": "?", "ignore": false}, - {"match": "*", "ignore": false}, - {"match": "**", "ignore": false}, - {"match": ".?", "ignore": false}, - {"match": ".*", "ignore": false}, - {"match": ".**", "ignore": false}, - {"match": "./*", "ignore": false}, - {"match": "./**", "ignore": false}, - {"match": "./**/*", "ignore": false}, - {"match": "./**/**", "ignore": false}, - {"match": "../*", "ignore": false}, - {"match": "../.*", "ignore": false}, - {"match": "../.**", "ignore": false}, - {"match": "../.**/*", "ignore": false}, - {"match": ".hfile", "ignore": false}, - {"match": "file", "ignore": false}, - {"match": "file"} - ]"# - .to_string(), - )]); - - let assets_temp_dir = create_temporary_assets_directory(files).unwrap(); - let assets_dir = assets_temp_dir.path().canonicalize().unwrap(); - let mut asset_descriptors = dbg!(gather_asset_descriptors(&[&assets_dir]).unwrap()); - - let mut expected_asset_descriptors = - vec![AssetDescriptor::default_from_path(&assets_dir, "file")]; - - expected_asset_descriptors.sort_by_key(|v| v.key.clone()); - asset_descriptors.sort_by_key(|v| v.key.clone()); - - assert_eq!(asset_descriptors, expected_asset_descriptors) - } - - #[test] - fn configuring_dotfiles_step_by_step() { - let files = HashMap::from([ - ( - Path::new(".ic-assets.json").to_path_buf(), - r#"[{"match": ".hidden-dir", "ignore": false}]"#.to_string(), - ), - ( - Path::new(".hidden-dir/.ic-assets.json").to_path_buf(), - r#"[ - {"match": ".hidden-dir-nested", "ignore": false}, - {"match": ".*", "ignore": false, "headers": {"A": "z"}}, - {"match": ".hfile", "headers": {"B": "y"}} - ]"# - .to_string(), - ), - ( - Path::new(".hidden-dir/.hidden-dir-nested/.ic-assets.json").to_path_buf(), - r#"[ - {"match": "*", "ignore": false, "headers": {"C": "x"}}, - {"match": ".hfile", "headers": {"D": "w"}} - ]"# - .to_string(), - ), - ]); - - let assets_temp_dir = create_temporary_assets_directory(files).unwrap(); - let assets_dir = assets_temp_dir.path().canonicalize().unwrap(); - let mut asset_descriptors = dbg!(gather_asset_descriptors(&[&assets_dir]).unwrap()); - - let mut expected_asset_descriptors = vec![ - AssetDescriptor::default_from_path(&assets_dir, "file"), - AssetDescriptor::default_from_path(&assets_dir, ".hidden-dir/.hfile") - .with_headers(HashMap::from([("B", "y"), ("A", "z")])), - AssetDescriptor::default_from_path(&assets_dir, ".hidden-dir/file"), - AssetDescriptor::default_from_path(&assets_dir, ".hidden-dir/.hidden-dir-nested/file") - .with_headers(HashMap::from([("A", "z"), ("C", "x")])), - AssetDescriptor::default_from_path( - &assets_dir, - ".hidden-dir/.hidden-dir-nested/.hfile", - ) - .with_headers(HashMap::from([("D", "w"), ("A", "z"), ("C", "x")])), - ]; - - expected_asset_descriptors.sort_by_key(|v| v.source.clone()); - asset_descriptors.sort_by_key(|v| v.source.clone()); - assert_eq!(asset_descriptors, expected_asset_descriptors) - } - - #[test] - fn include_only_a_specific_dotfile() { - let files = HashMap::from([ - ( - Path::new(".ic-assets.json").to_path_buf(), - r#"[ - {"match": ".hidden-dir", "ignore": false}, - {"match": "file", "ignore": true} - ]"# - .to_string(), - ), - ( - Path::new(".hidden-dir/.ic-assets.json").to_path_buf(), - r#"[ - {"match": "file", "ignore": true}, - {"match": ".hidden-dir-nested", "ignore": false} - ]"# - .to_string(), - ), - ( - Path::new(".hidden-dir/.hidden-dir-nested/.ic-assets.json").to_path_buf(), - r#"[ - {"match": "file", "ignore": true}, - {"match": ".hfile", "ignore": false, "headers": {"D": "w"}} - ]"# - .to_string(), - ), - ]); - - let assets_temp_dir = create_temporary_assets_directory(files).unwrap(); - let assets_dir = assets_temp_dir.path().canonicalize().unwrap(); - let mut asset_descriptors = dbg!(gather_asset_descriptors(&[&assets_dir]).unwrap()); - - let mut expected_asset_descriptors = vec![AssetDescriptor::default_from_path( - &assets_dir, - ".hidden-dir/.hidden-dir-nested/.hfile", - ) - .with_headers(HashMap::from([("D", "w")]))]; - - expected_asset_descriptors.sort_by_key(|v| v.source.clone()); - asset_descriptors.sort_by_key(|v| v.source.clone()); - assert_eq!(asset_descriptors, expected_asset_descriptors); - } - - #[test] - fn include_all_files_except_one() { - let files = HashMap::from([ - ( - Path::new(".ic-assets.json").to_path_buf(), - r#"[ - {"match": ".*", "ignore": false} - ]"# - .to_string(), - ), - ( - Path::new(".hidden-dir/.ic-assets.json").to_path_buf(), - r#"[ - {"match": "file", "ignore": true} - ]"# - .to_string(), - ), - ]); - - let assets_temp_dir = create_temporary_assets_directory(files).unwrap(); - let assets_dir = assets_temp_dir.path().canonicalize().unwrap(); - let mut asset_descriptors = dbg!(gather_asset_descriptors(&[&assets_dir]).unwrap()); - - let mut expected_asset_descriptors = vec![ - AssetDescriptor::default_from_path(&assets_dir, "file"), - AssetDescriptor::default_from_path(&assets_dir, ".hfile"), - AssetDescriptor::default_from_path(&assets_dir, ".hidden-dir-flat/file"), - AssetDescriptor::default_from_path(&assets_dir, ".hidden-dir-flat/.hfile"), - AssetDescriptor::default_from_path(&assets_dir, ".hidden-dir/.hfile"), - AssetDescriptor::default_from_path(&assets_dir, ".hidden-dir/.hidden-dir-nested/file"), - AssetDescriptor::default_from_path( - &assets_dir, - ".hidden-dir/.hidden-dir-nested/.hfile", - ), - ]; - - expected_asset_descriptors.sort_by_key(|v| v.source.clone()); - asset_descriptors.sort_by_key(|v| v.source.clone()); - assert_eq!(asset_descriptors, expected_asset_descriptors); - } - - #[test] - fn possible_to_reinclude_previously_ignored_file() { - let files = HashMap::from([ - ( - Path::new(".ic-assets.json").to_path_buf(), - r#"[ - {"match": ".hidden-dir-flat", "ignore": false}, - {"match": ".hidden-dir-flat/file", "ignore": true } - - ]"# - .to_string(), - ), - ( - Path::new(".hidden-dir-flat/.ic-assets.json").to_path_buf(), - r#"[ - {"match": "*", "ignore": false}, - {"match": "file", "ignore": false} - ]"# - .to_string(), - ), - ]); - - let assets_temp_dir = create_temporary_assets_directory(files).unwrap(); - let assets_dir = assets_temp_dir.path().canonicalize().unwrap(); - let mut asset_descriptors = dbg!(gather_asset_descriptors(&[&assets_dir]).unwrap()); - - let mut expected_asset_descriptors = vec![ - AssetDescriptor::default_from_path(&assets_dir, "file"), - AssetDescriptor::default_from_path(&assets_dir, ".hidden-dir-flat/.hfile"), - AssetDescriptor::default_from_path(&assets_dir, ".hidden-dir-flat/file"), - ]; - - expected_asset_descriptors.sort_by_key(|v| v.source.clone()); - asset_descriptors.sort_by_key(|v| v.source.clone()); - assert_eq!(asset_descriptors, expected_asset_descriptors); - } - - #[test] - /// It is not possible to include a file if its parent directory has been excluded - fn impossible_to_reinclude_file_from_already_ignored_directory() { - let files = HashMap::from([ - // additional, non-dot dirs and files - (Path::new("dir/file").to_path_buf(), "".to_string()), - (Path::new("anotherdir/file").to_path_buf(), "".to_string()), - ( - Path::new("anotherdir/.ic-assets.json").to_path_buf(), - r#"[ - {"match": "file", "ignore": false} - ]"# - .to_string(), - ), - // end of additional, non-dot dirs and files - ( - Path::new(".ic-assets.json").to_path_buf(), - r#"[ - {"match": "anotherdir", "ignore": true} - ]"# - .to_string(), - ), - ]); - - let assets_temp_dir = create_temporary_assets_directory(files).unwrap(); - let assets_dir = assets_temp_dir.path().canonicalize().unwrap(); - let mut asset_descriptors = dbg!(gather_asset_descriptors(&[&assets_dir]).unwrap()); - - let mut expected_asset_descriptors = vec![ - AssetDescriptor::default_from_path(&assets_dir, "file"), - AssetDescriptor::default_from_path(&assets_dir, "dir/file"), - ]; - - expected_asset_descriptors.sort_by_key(|v| v.source.clone()); - asset_descriptors.sort_by_key(|v| v.source.clone()); - assert_eq!(asset_descriptors, expected_asset_descriptors); - } - - #[test] - fn bonanza() { - let files = HashMap::from([ - // additional, non-dot dirs and files - (Path::new("dir/file").to_path_buf(), "".to_string()), - ( - Path::new("dir/.ic-assets.json").to_path_buf(), - r#"[ - {"match": "file", "headers": { "Access-Control-Allow-Origin": "null" }} - ]"# - .to_string(), - ), - (Path::new("anotherdir/file").to_path_buf(), "".to_string()), - ( - Path::new("anotherdir/.ic-assets.json").to_path_buf(), - r#"[ - {"match": "file", "cache": { "max_age": 42 }, "headers": null } - ]"# - .to_string(), - ), - // end of additional, non-dot dirs and files - ( - Path::new(".ic-assets.json").to_path_buf(), - r#"[ - {"match": "*", "cache": { "max_age": 11 }, "headers": { "X-Content-Type-Options": "nosniff" } }, - {"match": "**/.hfile", "ignore": false, "headers": { "X-Content-Type-Options": "*" }}, - {"match": ".hidden-dir-flat", "ignore": false }, - {"match": ".hidden-dir", "ignore": false } - - ]"# - .to_string(), - ), - ( - Path::new(".hidden-dir-flat/.ic-assets.json").to_path_buf(), - r#"[ - {"match": "*", "ignore": false, "headers": {"Cross-Origin-Resource-Policy": "same-origin"}}, - {"match": ".hfile", "ignore": true} - ]"# - .to_string(), - ), - ]); - - let assets_temp_dir = create_temporary_assets_directory(files).unwrap(); - let assets_dir = assets_temp_dir.path().canonicalize().unwrap(); - let mut asset_descriptors = gather_asset_descriptors(&[&assets_dir]).unwrap(); - - let mut expected_asset_descriptors = vec![ - AssetDescriptor::default_from_path(&assets_dir, ".hfile") - .with_headers(HashMap::from([("X-Content-Type-Options", "*")])) - .with_cache(CacheConfig { max_age: Some(11) }), - AssetDescriptor::default_from_path(&assets_dir, ".hidden-dir/.hfile") - .with_headers(HashMap::from([("X-Content-Type-Options", "*")])) - .with_cache(CacheConfig { max_age: Some(11) }), - AssetDescriptor::default_from_path(&assets_dir, ".hidden-dir/file") - .with_headers(HashMap::from([("X-Content-Type-Options", "nosniff")])) - .with_cache(CacheConfig { max_age: Some(11) }), - AssetDescriptor::default_from_path(&assets_dir, ".hidden-dir-flat/file") - .with_headers(HashMap::from([("X-Content-Type-Options", "nosniff")])) - .with_headers(HashMap::from([( - "Cross-Origin-Resource-Policy", - "same-origin", - )])) - .with_cache(CacheConfig { max_age: Some(11) }), - AssetDescriptor::default_from_path(&assets_dir, "anotherdir/file") - .with_cache(CacheConfig { max_age: Some(42) }), - AssetDescriptor::default_from_path(&assets_dir, "dir/file") - .with_headers(HashMap::from([("X-Content-Type-Options", "nosniff")])) - .with_headers(HashMap::from([("Access-Control-Allow-Origin", "null")])) - .with_cache(CacheConfig { max_age: Some(11) }), - AssetDescriptor::default_from_path(&assets_dir, "file") - .with_cache(CacheConfig { max_age: Some(11) }) - .with_headers(HashMap::from([("X-Content-Type-Options", "nosniff")])), - ]; - - expected_asset_descriptors.sort_by_key(|v| v.source.clone()); - asset_descriptors.sort_by_key(|v| v.source.clone()); - assert_eq!(dbg!(asset_descriptors), expected_asset_descriptors); - } -} diff --git a/ic-asset/src/upload.rs b/ic-asset/src/upload.rs deleted file mode 100644 index 111fc763..00000000 --- a/ic-asset/src/upload.rs +++ /dev/null @@ -1,71 +0,0 @@ -use crate::asset_canister::batch::{commit_batch, create_batch}; -use crate::asset_canister::list::list_assets; -use crate::asset_canister::protocol::{AssetDetails, BatchOperationKind}; -use crate::asset_config::AssetConfig; -use crate::operations::{ - create_new_assets, delete_incompatible_assets, set_encodings, unset_obsolete_encodings, -}; -use crate::params::CanisterCallParams; -use crate::plumbing::{make_project_assets, AssetDescriptor, ProjectAsset}; -use ic_utils::Canister; -use std::collections::HashMap; -use std::path::PathBuf; -use std::time::Duration; - -/// Upload the specified files -pub async fn upload( - canister: &Canister<'_>, - timeout: Duration, - files: HashMap, -) -> anyhow::Result<()> { - let asset_descriptors: Vec = files - .iter() - .map(|x| AssetDescriptor { - source: x.1.clone(), - key: x.0.clone(), - config: AssetConfig::default(), - }) - .collect(); - - let canister_call_params = CanisterCallParams { canister, timeout }; - - let container_assets = list_assets(&canister_call_params).await?; - - println!("Starting batch."); - - let batch_id = create_batch(&canister_call_params).await?; - - println!("Staging contents of new and changed assets:"); - - let project_assets = make_project_assets( - &canister_call_params, - &batch_id, - asset_descriptors, - &container_assets, - ) - .await?; - - let operations = assemble_upload_operations(project_assets, container_assets); - - println!("Committing batch."); - - commit_batch(&canister_call_params, &batch_id, operations).await?; - - Ok(()) -} - -fn assemble_upload_operations( - project_assets: HashMap, - container_assets: HashMap, -) -> Vec { - let mut container_assets = container_assets; - - let mut operations = vec![]; - - delete_incompatible_assets(&mut operations, &project_assets, &mut container_assets); - create_new_assets(&mut operations, &project_assets, &container_assets); - unset_obsolete_encodings(&mut operations, &project_assets, &container_assets); - set_encodings(&mut operations, project_assets); - - operations -} diff --git a/icx-asset/Cargo.toml b/icx-asset/Cargo.toml deleted file mode 100644 index 2f2d361f..00000000 --- a/icx-asset/Cargo.toml +++ /dev/null @@ -1,41 +0,0 @@ -[package] -name = "icx-asset" -version = "0.20.0" -authors = ["DFINITY Stiftung "] -edition = "2021" -description = "CLI tool to manage assets on an asset canister on the Internet Computer." -homepage = "https://docs.rs/icx-asset" -documentation = "https://docs.rs/icx-asset" -repository = "https://github.com/dfinity/agent-rs" -license = "Apache-2.0" -readme = "README.md" -categories = ["command-line-interface"] -keywords = ["internet-computer", "agent", "icp", "dfinity", "asset"] -include = ["src", "Cargo.toml", "../LICENSE", "README.md"] -rust-version = "1.60.0" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -anyhow = "1.0.34" -candid = "0.7.15" -chrono = "0.4.19" -clap = { version = "3.0.14", features = ["derive", "cargo"] } -delay = "0.3.1" -garcon = "0.2.2" -humantime = "2.0.1" -ic-agent = { path = "../ic-agent", version = "0.20", default-features = false } -ic-asset = { path = "../ic-asset", version = "0.20" } -ic-utils = { path = "../ic-utils", version = "0.20" } -libflate = "1.2.0" -num-traits = "0.2" -pem = "1.0.1" -serde = "1.0.115" -serde_bytes = "0.11.5" -serde_json = "1.0.57" -tokio = { version = "1.2.0", features = ["full", "rt"] } -thiserror = "1.0.24" -walkdir = "2.3.1" - -[dev-dependencies] -ic-agent = { path = "../ic-agent", version = "0.20", features = ["reqwest"] } diff --git a/icx-asset/README.md b/icx-asset/README.md index 7270c012..4cb65793 100644 --- a/icx-asset/README.md +++ b/icx-asset/README.md @@ -1,40 +1,3 @@ -# `icx-asset` -A command line tool to manage an asset storage canister. +# Notice -## icx-asset sync - -Synchronize one or more directories to an asset canister. - -Usage: `icx-asset sync ...` - -Example: -``` -# same asset synchronization as dfx deploy for a default project, if you've already run dfx build -$ icx-asset --pem ~/.config/dfx/identity/default/identity.pem sync src/prj_assets/assets dist/prj_assets -``` - -## icx-asset ls - -List assets in the asset canister. - -## icx-asset upload - -Usage: `icx-asset upload [=] [[=] ...]` - -Examples: - -``` -# upload a single file as /a.txt -$ icx-asset upload a.txt - -# upload a single file, a.txt, under another name -$ icx-asset upload /b.txt=a.txt - -# upload a directory and its contents as /some-dir/* -$ icx-asset upload some-dir - -# Similar to synchronization with dfx deploy, but without deleting anything: -$ icx-asset upload /=src//assets - - -``` \ No newline at end of file +The `icx-asset` crate has been moved to the [sdk](https://github.com/dfinity/sdk) repo. diff --git a/icx-asset/src/commands/list.rs b/icx-asset/src/commands/list.rs deleted file mode 100644 index 7b608503..00000000 --- a/icx-asset/src/commands/list.rs +++ /dev/null @@ -1,56 +0,0 @@ -use crate::support::Result; -use candid::{CandidType, Int, Nat}; -use ic_utils::call::SyncCall; -use ic_utils::Canister; - -use num_traits::ToPrimitive; -use serde::Deserialize; -use std::time::SystemTime; - -pub async fn list(canister: &Canister<'_>) -> Result { - #[derive(CandidType, Deserialize)] - struct Encoding { - modified: Int, - content_encoding: String, - sha256: Option>, - length: Nat, - } - - #[derive(CandidType, Deserialize)] - struct ListEntry { - key: String, - content_type: String, - encodings: Vec, - } - - #[derive(CandidType, Deserialize)] - struct EmptyRecord {} - - let (entries,): (Vec,) = canister - .query_("list") - .with_arg(EmptyRecord {}) - .build() - .call() - .await?; - - use chrono::offset::Local; - use chrono::DateTime; - - for entry in entries { - for encoding in entry.encodings { - let modified = encoding.modified; - let modified = SystemTime::UNIX_EPOCH - + std::time::Duration::from_nanos(modified.0.to_u64().unwrap()); - - eprintln!( - "{:>20} {:>15} {:50} ({}, {})", - DateTime::::from(modified).format("%F %X"), - encoding.length.0, - entry.key, - entry.content_type, - encoding.content_encoding - ); - } - } - Ok(()) -} diff --git a/icx-asset/src/commands/mod.rs b/icx-asset/src/commands/mod.rs deleted file mode 100644 index a889ae5b..00000000 --- a/icx-asset/src/commands/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod list; -pub mod sync; -pub mod upload; diff --git a/icx-asset/src/commands/sync.rs b/icx-asset/src/commands/sync.rs deleted file mode 100644 index 131b9201..00000000 --- a/icx-asset/src/commands/sync.rs +++ /dev/null @@ -1,15 +0,0 @@ -use ic_utils::Canister; -use std::path::Path; - -use crate::{support, SyncOpts}; -use std::time::Duration; - -pub(crate) async fn sync( - canister: &Canister<'_>, - timeout: Duration, - o: &SyncOpts, -) -> support::Result { - let dirs: Vec<&Path> = o.directory.iter().map(|d| d.as_path()).collect(); - ic_asset::sync(canister, &dirs, timeout).await?; - Ok(()) -} diff --git a/icx-asset/src/commands/upload.rs b/icx-asset/src/commands/upload.rs deleted file mode 100644 index bc96eb1b..00000000 --- a/icx-asset/src/commands/upload.rs +++ /dev/null @@ -1,59 +0,0 @@ -use crate::{support, UploadOpts}; -use ic_utils::Canister; -use std::collections::HashMap; -use std::path::PathBuf; -use std::str::FromStr; -use std::time::Duration; -use walkdir::WalkDir; - -pub(crate) async fn upload(canister: &Canister<'_>, opts: &UploadOpts) -> support::Result { - let key_map = get_key_map(&opts.files)?; - ic_asset::upload(canister, Duration::from_secs(500), key_map).await?; - Ok(()) -} - -fn get_key_map(files: &[String]) -> anyhow::Result> { - let mut key_map: HashMap = HashMap::new(); - - for arg in files { - let (key, source): (String, PathBuf) = { - if let Some(index) = arg.find('=') { - ( - arg[..index].to_string(), - PathBuf::from_str(&arg[index + 1..])?, - ) - } else { - let source = PathBuf::from_str(&arg.clone())?; - let key = format!("/{}", source.file_name().unwrap().to_string_lossy()); - // or if we want to retain relative paths: - // let key = if source.is_absolute() { - // format!("/{}", source.file_name().unwrap().to_string_lossy()) - // } else { - // format!("/{}", arg.clone()) - // }; - (key, source) - } - }; - - if source.is_file() { - key_map.insert(key, source); - } else { - for p in WalkDir::new(source.clone()) - .into_iter() - .filter_map(std::result::Result::ok) - .filter(|e| !e.file_type().is_dir()) - { - let p = p.path().to_path_buf(); - let relative = p.strip_prefix(&source).expect("cannot strip prefix"); - let mut key = key.clone(); - if !key.ends_with('/') { - key.push('/'); - } - key.push_str(relative.to_string_lossy().as_ref()); - key_map.insert(key, p); - } - } - } - - Ok(key_map) -} diff --git a/icx-asset/src/main.rs b/icx-asset/src/main.rs deleted file mode 100644 index 20414140..00000000 --- a/icx-asset/src/main.rs +++ /dev/null @@ -1,142 +0,0 @@ -mod commands; -mod support; - -use crate::commands::list::list; -use crate::commands::sync::sync; -use candid::Principal; -use clap::{crate_authors, crate_version, Parser}; -use ic_agent::identity::{AnonymousIdentity, BasicIdentity, Secp256k1Identity}; -use ic_agent::{agent, Agent, Identity}; - -use crate::commands::upload::upload; -use std::path::PathBuf; -use std::time::Duration; - -const DEFAULT_IC_GATEWAY: &str = "https://ic0.app"; - -#[derive(Parser)] -#[clap( - version = crate_version!(), - author = crate_authors!(), - propagate_version(true), -)] -struct Opts { - /// Some input. Because this isn't an Option it's required to be used - #[clap(long, default_value = "http://localhost:8000/")] - replica: String, - - /// An optional PEM file to read the identity from. If none is passed, - /// a random identity will be created. - #[clap(long)] - pem: Option, - - /// An optional field to set the expiry time on requests. Can be a human - /// readable time (like `100s`) or a number of seconds. - #[clap(long)] - ttl: Option, - - #[clap(subcommand)] - subcommand: SubCommand, -} - -#[derive(Parser)] -enum SubCommand { - /// List keys from the asset canister. - #[clap(name = "ls")] - List(ListOpts), - - /// Synchronize a directory to the asset canister - Sync(SyncOpts), - - /// Uploads an asset to an asset canister. - Upload(UploadOpts), -} - -#[derive(Parser)] -struct ListOpts { - /// The canister ID. - #[clap()] - canister_id: String, -} - -#[derive(Parser)] -struct SyncOpts { - /// The canister ID. - #[clap()] - canister_id: String, - - /// The directories to synchronize - #[clap()] - directory: Vec, -} - -#[derive(Parser)] -struct UploadOpts { - /// The asset canister ID to manage. - #[clap()] - canister_id: String, - - /// Files or folders to send. - #[clap()] - files: Vec, -} - -fn create_identity(maybe_pem: Option) -> Box { - if let Some(pem_path) = maybe_pem { - if let Ok(secp256k_identity) = Secp256k1Identity::from_pem_file(&pem_path) { - Box::new(secp256k_identity) - } else { - Box::new(BasicIdentity::from_pem_file(pem_path).expect("Could not read the key pair.")) - } - } else { - Box::new(AnonymousIdentity) - } -} - -#[tokio::main(flavor = "multi_thread", worker_threads = 10)] -async fn main() -> support::Result { - let opts: Opts = Opts::parse(); - - let ttl: std::time::Duration = opts - .ttl - .map(|ht| ht.into()) - .unwrap_or_else(|| Duration::from_secs(60 * 5)); // 5 minutes is max ingress timeout - - let agent = Agent::builder() - .with_transport( - agent::http_transport::ReqwestHttpReplicaV2Transport::create(opts.replica.clone())?, - ) - .with_boxed_identity(create_identity(opts.pem)) - .build()?; - - let normalized_replica = opts.replica.strip_suffix('/').unwrap_or(&opts.replica); - if normalized_replica != DEFAULT_IC_GATEWAY { - agent.fetch_root_key().await?; - } - - match &opts.subcommand { - SubCommand::List(o) => { - let canister = ic_utils::Canister::builder() - .with_agent(&agent) - .with_canister_id(Principal::from_text(&o.canister_id)?) - .build()?; - list(&canister).await?; - } - SubCommand::Sync(o) => { - let canister = ic_utils::Canister::builder() - .with_agent(&agent) - .with_canister_id(Principal::from_text(&o.canister_id)?) - .build()?; - sync(&canister, ttl, o).await?; - } - SubCommand::Upload(o) => { - let canister = ic_utils::Canister::builder() - .with_agent(&agent) - .with_canister_id(Principal::from_text(&o.canister_id)?) - .build()?; - upload(&canister, o).await?; - } - } - - Ok(()) -} diff --git a/icx-asset/src/support.rs b/icx-asset/src/support.rs deleted file mode 100644 index e79fb404..00000000 --- a/icx-asset/src/support.rs +++ /dev/null @@ -1 +0,0 @@ -pub type Result = std::result::Result>;