chore: check each feature works properly (#1695)

It is hard to maintain features list manually, so use cargo-hack's
`--each-feature` flag. And cargo-hack provides a workaround for an issue
that dev-dependencies leaking into normal build (`--no-dev-deps` flag),
so removed own ci tool.

Also, compared to running tests on all features, there is not much
advantage in running tests on each feature, so only the default features
and all features are tested.
If the behavior changes depending on the feature, we need to test it as
another job in CI.

Author: taiki-e

Cargo.toml

@@ -9,6 +9,5 @@ members = [
 
   # Internal
   "examples",
-  "tests-build",
   "tests-integration",
 ]

azure-pipelines.yml

@@ -25,20 +25,8 @@ jobs:
     displayName: Test tokio
     cross: true
     crates:
-      tokio:
-        - fs
-        - io
-        - io-util
-        - net-driver
-        - process
-        - rt-full
-        - signal
-        - sync
-        - tcp
-        - timer
-        - udp
-        - uds
-      tests-integration: []
+      - tokio
+      - tests-integration
 
 # Test crates that are NOT platform specific
 - template: ci/azure-test-stable.yml
@@ -47,29 +35,10 @@ jobs:
     displayName: Test sub crates -
     rust: beta
     crates:
-      tokio-macros: []
-      tokio-test: []
-      tokio-util: []
-      examples: []
-
-# Test compilation failure
-# Disable pending: https://github.com/tokio-rs/tokio/pull/1695#issuecomment-547045383
-# - template: ci/azure-test-stable.yml
-#   parameters:
-#     name: test_features
-#     displayName: Test feature flags
-#     rust: beta
-#     crates:
-#       tests-build:
-#         # - tokio-executor
-#         # - executor-without-current-thread
-#         # - macros-invalid-input
-#         # - net-no-features
-#         # - net-with-tcp
-#         # - net-with-udp
-#         # - net-with-uds
-#         # - tokio-no-features
-#         # - tokio-with-net
+      - tokio-macros
+      - tokio-test
+      - tokio-util
+      - examples
 
 # Run loom tests
 - template: ci/azure-loom.yml

ci/azure-test-stable.yml

@@ -20,6 +20,11 @@ jobs:
       # rust_version: stable
       rust_version: ${{ parameters.rust }}
 
+  - script: cargo install cargo-hack
+    displayName: Install cargo-hack
+    # FIXME(taiki-e): many features of current cargo-hack are not available on windows: https://github.com/taiki-e/cargo-hack/issues/3
+    condition: not(eq(variables['Agent.OS'], 'Windows_NT'))
+
   - template: azure-is-release.yml
 
   - ${{ each crate in parameters.crates }}:
@@ -28,17 +33,28 @@ jobs:
       env:
         LOOM_MAX_PREEMPTIONS: 2
         CI: 'True'
-      displayName: ${{ crate.key }} - cargo test
-      workingDirectory: $(Build.SourcesDirectory)/${{ crate.key }}
-
-    # Run with each specified feature
-    - ${{ each feature in crate.value }}:
-      - script: cargo test --no-default-features --features ${{ feature }}
-        env:
-          LOOM_MAX_PREEMPTIONS: 2
-          CI: 'True'
-        displayName: ${{ crate.key }} - cargo test --features ${{ feature }}
-        workingDirectory: $(Build.SourcesDirectory)/${{ crate.key }}
+      displayName: ${{ crate }} - cargo test
+      workingDirectory: $(Build.SourcesDirectory)/${{ crate }}
+
+    # Run with all crate features
+    - script: cargo test --all-features
+      env:
+        LOOM_MAX_PREEMPTIONS: 2
+        CI: 'True'
+      displayName: ${{ crate }} - cargo test --all-features
+      workingDirectory: $(Build.SourcesDirectory)/${{ crate }}
+
+    # Check each specified feature works properly
+    # * --each-feature - run for each feature which includes --no-default-features and default features of package
+    # * --no-dev-deps - build without dev-dependencies to avoid https://github.com/rust-lang/cargo/issues/4866
+    - script: cargo hack check --each-feature --no-dev-deps
+      env:
+        LOOM_MAX_PREEMPTIONS: 2
+        CI: 'True'
+      displayName: ${{ crate }} - cargo hack check --each-feature
+      workingDirectory: $(Build.SourcesDirectory)/${{ crate }}
+      # FIXME(taiki-e): many features of current cargo-hack are not available on windows: https://github.com/taiki-e/cargo-hack/issues/3
+      condition: not(eq(variables['Agent.OS'], 'Windows_NT'))
 
   - template: azure-patch-crates.yml
 
@@ -48,14 +64,25 @@ jobs:
       env:
         LOOM_MAX_PREEMPTIONS: 2
         CI: 'True'
-      displayName: ${{ crate.key }} - cargo test
-      workingDirectory: $(Build.SourcesDirectory)/${{ crate.key }}
-
-    # Run with each specified feature
-    - ${{ each feature in crate.value }}:
-      - script: cargo test --no-default-features --features ${{ feature }}
-        env:
-          LOOM_MAX_PREEMPTIONS: 2
-          CI: 'True'
-        displayName: ${{ crate.key }} - cargo test --features ${{ feature }}
-        workingDirectory: $(Build.SourcesDirectory)/${{ crate.key }}
+      displayName: ${{ crate }} - cargo test
+      workingDirectory: $(Build.SourcesDirectory)/${{ crate }}
+
+    # Run with all crate features
+    - script: cargo test --all-features
+      env:
+        LOOM_MAX_PREEMPTIONS: 2
+        CI: 'True'
+      displayName: ${{ crate }} - cargo test --all-features
+      workingDirectory: $(Build.SourcesDirectory)/${{ crate }}
+
+    # Check each specified feature works properly
+    # * --each-feature - run for each feature which includes --no-default-features and default features of package
+    # * --no-dev-deps - build without dev-dependencies to avoid https://github.com/rust-lang/cargo/issues/4866
+    - script: cargo hack check --each-feature --no-dev-deps
+      env:
+        LOOM_MAX_PREEMPTIONS: 2
+        CI: 'True'
+      displayName: ${{ crate }} - cargo hack check --each-feature
+      workingDirectory: $(Build.SourcesDirectory)/${{ crate }}
+      # FIXME(taiki-e): many features of current cargo-hack are not available on windows: https://github.com/taiki-e/cargo-hack/issues/3
+      condition: not(eq(variables['Agent.OS'], 'Windows_NT'))