diff --git a/configure b/configure index 45b2ed0..433d1a8 100755 --- a/configure +++ b/configure @@ -4,8 +4,9 @@ set -e CMAKE_FLAGS= # defaults -# WITH_BLAS=1 +USE_BLAS=0 USE_OPENCV=0 +BUILD_TESTS=1 add_cmake_flag() { echo "usinig $1=$2" @@ -18,15 +19,15 @@ parse_args() { --prefix=*) PREFIX="${i#*=}" ;; + --use-blas) + USE_BLAS=1 + ;; --use-opencv) USE_OPENCV=1 ;; - # --with-blas) - # WITH_BLAS=1 - # ;; - # --without-blas) - # WITH_BLAS= - # ;; + --no-tests) + BUILD_TESTS=0 + ;; *) echo "unknown argument $1" exit @@ -65,12 +66,8 @@ add_cmake_flags() { # add_cmake_flag USE_OPT 1 add_cmake_flag USE_OPENCV ${USE_OPENCV} - # add_cmake_flag USE_OPENBLAS 1 - # add_cmake_flag BUILD_TESTS 0 - - # if [ ! -z ${WITH_BLAS} ]; then - # add_cmake_flag USE_BLAS 1 - # fi + add_cmake_flag USE_OPENBLAS ${USE_BLAS} + add_cmake_flag BUILD_TESTS ${BUILD_TESTS} } parse_args $@ diff --git a/include/nn/bits/layers/conv.hpp b/include/nn/bits/layers/conv.hpp index 8b85e3c..7c58e59 100644 --- a/include/nn/bits/layers/conv.hpp +++ b/include/nn/bits/layers/conv.hpp @@ -33,6 +33,12 @@ class conv_layer_trait : public ops::conv_trait { } + conv_layer_trait(const ksize_t &ksize, size_t n_filters, + const conv_trait &trait) + : conv_trait(trait), ksize_(ksize), n_filters_(n_filters) + { + } + template shape<4> filter_shape(const shape<4> &x) const { @@ -63,8 +69,8 @@ class conv : public conv_layer_trait { auto w = ops::new_parameter>( filter_shape(x.shape()), w_init); - auto y = ops::new_result>( - conv_op(padding_, stride_, rate_), x, *w); + auto y = ops::new_result>(conv_op(h_trait_, w_trait_), + x, *w); Act()(ref(*y), view(*y)); return make_layer(y, w); @@ -86,8 +92,8 @@ class conv : public conv_layer_trait { auto w = ops::new_parameter>( filter_shape(x.shape()), w_init); - auto y = ops::new_result>( - conv_op(padding_, stride_, rate_), x, *w); + auto y = ops::new_result>(conv_op(h_trait_, w_trait_), + x, *w); using add_bias = nn::ops::apply_bias>; auto b = ops::new_parameter>(bias_shape(x.shape()), diff --git a/include/nn/bits/ops/conv.hpp b/include/nn/bits/ops/conv.hpp index 19a23fd..06bac1a 100644 --- a/include/nn/bits/ops/conv.hpp +++ b/include/nn/bits/ops/conv.hpp @@ -21,7 +21,18 @@ template class linear_conv_trait const dim_t rate_; const dim_t stride_; + using sample_t = linear_sample_trait; + public: + using padding_t = typename sample_t::padding_t; + + static padding_t padding(int p) { return padding_t(p, p); } + + static padding_t padding(int left, int right) + { + return padding_t(left, right); + }; + linear_conv_trait() : linear_conv_trait(default_pad_lr) {} linear_conv_trait(dim_t pad_lr) : linear_conv_trait(pad_lr, default_stride) @@ -34,53 +45,82 @@ template class linear_conv_trait } linear_conv_trait(dim_t pad_lr, dim_t stride, dim_t rate) - : pad_l_(pad_lr), pad_r_(pad_lr), rate_(rate), stride_(stride) + : linear_conv_trait(paddint(pad_lr), stride, rate) { } - dim_t operator()(dim_t n, dim_t k) const + linear_conv_trait(const padding_t &pad, dim_t stride, dim_t rate) + : pad_l_(std::get<0>(pad.dims)), + pad_r_(std::get<1>(pad.dims)), + rate_(rate), + stride_(stride) { - return linear_sample_trait(k, stride_, rate_, pad_l_, pad_r_)(n); } + + sample_t get_sample(dim_t ksize) const + { + return sample_t(ksize, stride_, rate_, pad_l_, pad_r_); + } + + dim_t operator()(dim_t n, dim_t k) const { return get_sample(k)(n); } }; template class conv_trait; template <> class conv_trait { - using conv_trait_1d_t = linear_conv_trait; + using dim_t = size_t; + using conv_trait_1d_t = linear_conv_trait; + using padding_1d_t = conv_trait_1d_t::padding_t; protected: - struct padding_trait; struct stride_trait; struct rate_trait; - using padding_t = std::experimental::new_type, padding_trait>; + using padding_t = std::array; + using stride_t = std::experimental::new_type, stride_trait>; using rate_t = std::experimental::new_type, rate_trait>; - static constexpr auto default_padding = padding_t(0, 0); static constexpr auto default_stride = stride_t(1, 1); static constexpr auto default_rate = rate_t(1, 1); - const padding_t padding_; - const stride_t stride_; - const rate_t rate_; - const conv_trait_1d_t h_trait_; const conv_trait_1d_t w_trait_; + static padding_t default_padding() { return padding(0, 0); } + public: - static padding_t padding(int r, int s) { return padding_t(r, s); }; + static padding_1d_t padding_1d(dim_t p) { return padding_1d_t(p, p); } + + static padding_1d_t padding_1d(dim_t left, dim_t right) + { + return padding_1d_t(left, right); + } + + static padding_t padding(dim_t r, dim_t s) + { + return padding(padding_1d(r), padding_1d(s)); + }; + + static padding_t padding(const padding_1d_t &r, const padding_1d_t &s) + { + return {r, s}; + }; + static stride_t stride(int r, int s) { return stride_t(r, s); }; - conv_trait() : conv_trait(default_padding) {} + static rate_t rate(int r, int s) { return rate_t(r, s); }; + + conv_trait() : conv_trait(default_padding()) {} conv_trait(const padding_t &padding) : conv_trait(padding, default_stride) { } - conv_trait(const stride_t &stride) : conv_trait(default_padding, stride) {} + conv_trait(const stride_t &stride) : conv_trait(default_padding(), stride) + { + } conv_trait(const padding_t &padding, const stride_t &stride) : conv_trait(padding, stride, default_rate) @@ -89,11 +129,13 @@ template <> class conv_trait conv_trait(const padding_t &padding, const stride_t &stride, const rate_t &rate) - : padding_(padding), - stride_(stride), - rate_(rate), - h_trait_(padding.dims[0], stride.dims[0], rate.dims[0]), - w_trait_(padding.dims[1], stride.dims[1], rate.dims[1]) + : h_trait_(padding[0], stride.dims[0], rate.dims[0]), + w_trait_(padding[1], stride.dims[1], rate.dims[1]) + { + } + + conv_trait(const conv_trait_1d_t &h_trait, const conv_trait_1d_t &w_trait) + : h_trait_(h_trait), w_trait_(w_trait) { } @@ -139,10 +181,7 @@ template <> class conv : public conv_trait using upper_op = im2col; const auto upper = internal::make_batched( - upper_op(upper_op::ksize(r, s), - upper_op::padding(padding_.dims[0], padding_.dims[1]), - upper_op::stride(stride_.dims[0], stride_.dims[1]), - upper_op::rate(rate_.dims[0], rate_.dims[1]))); + upper_op(h_trait_.get_sample(r), w_trait_.get_sample(s))); ttl::tensor x_upper(upper(x.shape())); upper(ref(x_upper), view(x)); @@ -172,10 +211,8 @@ template <> class conv : public conv_trait using upper_op = im2col; const auto [r, s] = filter_shape(y.shape()).dims; const auto upper = internal::make_batched( - upper_op(upper_op::ksize(r, s), - upper_op::padding(padding_.dims[0], padding_.dims[1]), - upper_op::stride(stride_.dims[0], stride_.dims[1]), - upper_op::rate(rate_.dims[0], rate_.dims[1]))); + upper_op(h_trait_.get_sample(r), w_trait_.get_sample(s))); + ttl::tensor x_upper(upper(x.shape().template subshape<1>())); const auto n = batch_size(z.shape()); for (auto l : range(n)) { diff --git a/include/nn/bits/ops/im2col.hpp b/include/nn/bits/ops/im2col.hpp index a9d3851..d68f9ee 100644 --- a/include/nn/bits/ops/im2col.hpp +++ b/include/nn/bits/ops/im2col.hpp @@ -1,82 +1,16 @@ #pragma once -#include -#include #include #include #include -#include +#include namespace nn::ops { template class im2col_trait; -template <> class im2col_trait +template <> class im2col_trait : public multi_linear_sample_trait<2, size_t> { - protected: - struct ksize_trait; - struct padding_trait; - struct stride_trait; - struct rate_trait; - - using ksize_t = std::experimental::new_type, ksize_trait>; - using padding_t = std::experimental::new_type, padding_trait>; - using stride_t = std::experimental::new_type, stride_trait>; - using rate_t = std::experimental::new_type, rate_trait>; - - static constexpr auto default_padding = padding_t(0, 0); - static constexpr auto default_stride = stride_t(1, 1); - static constexpr auto default_rate = rate_t(1, 1); - - using sample_t = linear_sample_trait; - - const sample_t h_sample_; - const sample_t w_sample_; - - ksize_t get_ksize() const - { - return ksize_t(h_sample_.ksize_, w_sample_.ksize_); - } - - public: - static ksize_t ksize(int r, int s) { return ksize_t(r, s); }; - static padding_t padding(int r, int s) { return padding_t(r, s); }; - static stride_t stride(int r, int s) { return stride_t(r, s); }; - static rate_t rate(int r, int s) { return rate_t(r, s); }; - - im2col_trait(const ksize_t &ksize) - : im2col_trait(ksize, default_padding, default_stride) - { - } - - im2col_trait(const ksize_t &ksize, const padding_t &padding) - : im2col_trait(ksize, padding, default_stride) - { - } - - im2col_trait(const ksize_t &ksize, const stride_t &stride) - : im2col_trait(ksize, default_padding, stride) - { - } - - im2col_trait(const ksize_t &ksize, const padding_t &padding, - const stride_t &stride) - : im2col_trait(ksize, padding, stride, default_rate) - { - } - - im2col_trait(const ksize_t &ksize, const padding_t &padding, - const stride_t &stride, const rate_t &rate) - : h_sample_(ksize.dims[0], stride.dims[0], rate.dims[0], - padding.dims[0]), - w_sample_(ksize.dims[1], stride.dims[1], rate.dims[1], - padding.dims[1]) - { - } - - shape<2> operator()(const shape<2> &x) const - { - return shape<2>(h_sample_(x.dims[0]), w_sample_(x.dims[1])); - } + using multi_linear_sample_trait::multi_linear_sample_trait; }; template class im2col; @@ -98,6 +32,9 @@ template <> class im2col : public im2col_trait const auto [h, w] = x.shape().dims; const auto [h_, w_, r, s] = y.shape().dims; + const sample_t &h_sample_ = std::get<0>(samples_); + const sample_t &w_sample_ = std::get<1>(samples_); + for (const auto i_ : range(h_)) { for (const auto j_ : range(w_)) { for (const auto u : range(r)) { @@ -134,6 +71,9 @@ template <> class im2col : public im2col_trait const auto [h, w] = x.shape().dims; const auto [r, s, h_, w_] = y.shape().dims; + const sample_t &h_sample_ = std::get<0>(samples_); + const sample_t &w_sample_ = std::get<1>(samples_); + for (const auto u : range(r)) { for (const auto v : range(s)) { for (const auto i_ : range(h_)) { @@ -162,8 +102,9 @@ template <> class im2col : public im2col_trait shape<5> operator()(const shape<3> &x) const { const auto [r, s] = get_ksize().dims; - return shape<5>(h_sample_(x.dims[0]), w_sample_(x.dims[1]), r, s, - x.dims[2]); + const auto [h, w, c] = x.dims; + const auto [h_, w_] = im2col_trait::operator()(shape<2>(h, w)).dims; + return shape<5>(h_, w_, r, s, c); } template @@ -174,6 +115,9 @@ template <> class im2col : public im2col_trait const auto [h_, w_, r, s, _c] = y.shape().dims; contract_assert(_c == c); + const sample_t &h_sample_ = std::get<0>(samples_); + const sample_t &w_sample_ = std::get<1>(samples_); + for (const auto i_ : range(h_)) { for (const auto j_ : range(w_)) { for (const auto u : range(r)) { diff --git a/include/nn/bits/ops/linear_sample.hpp b/include/nn/bits/ops/linear_sample.hpp index e2c9442..d12c200 100644 --- a/include/nn/bits/ops/linear_sample.hpp +++ b/include/nn/bits/ops/linear_sample.hpp @@ -1,5 +1,5 @@ #pragma once -#include +#include /*! \begin{definition} @@ -40,19 +40,29 @@ namespace nn::ops template class linear_sample_trait { - static constexpr dim_t default_rate = 1; - static constexpr dim_t default_stride = 1; - static constexpr dim_t default_pad_lr = 0; - const dim_t pad_l_; // TODO: make it template parameter const dim_t pad_r_; // TODO: make it template parameter - public: - // FIXME: const dim_t rate_; const dim_t stride_; const dim_t ksize_; + struct padding_trait; + + public: + static constexpr dim_t default_rate = 1; + static constexpr dim_t default_stride = 1; + static constexpr dim_t default_pad_lr = 0; + + using padding_t = std::experimental::new_type, padding_trait>; + + static padding_t padding(dim_t p) { return padding_t(p, p); } + + static padding_t padding(dim_t left, dim_t right) + { + return padding_t(left, right); + }; + public: linear_sample_trait(dim_t ksize) : linear_sample_trait(ksize, default_stride) @@ -70,14 +80,21 @@ template class linear_sample_trait } linear_sample_trait(dim_t ksize, dim_t stride, dim_t rate, dim_t pad_lr) - : linear_sample_trait(ksize, stride, rate, pad_lr, pad_lr) + : linear_sample_trait(ksize, stride, rate, padding(pad_lr)) { } linear_sample_trait(dim_t ksize, dim_t stride, dim_t rate, dim_t pad_l, dim_t pad_r) - : pad_l_(pad_l), - pad_r_(pad_r), + : linear_sample_trait(ksize, stride, rate, padding(pad_l, pad_r)) + { + // TODO: deprecate it + } + + linear_sample_trait(dim_t ksize, dim_t stride, dim_t rate, + const padding_t &pad) + : pad_l_(std::get<0>(pad.dims)), + pad_r_(std::get<1>(pad.dims)), rate_(rate), stride_(stride), ksize_(ksize) @@ -85,10 +102,14 @@ template class linear_sample_trait contract_assert(rate_ >= 1); contract_assert(stride_ >= 1); contract_assert(ksize_ >= 1); - contract_assert(pad_l >= 0); - contract_assert(pad_r >= 0); + contract_assert(pad_l_ >= 0); + contract_assert(pad_r_ >= 0); } + dim_t get_ksize() const { return ksize_; } + + dim_t get_stride() const { return stride_; } + /*! Compute the output size from input size. */ dim_t operator()(dim_t n) const { @@ -125,4 +146,175 @@ template class linear_sample_trait dim_t unpad(dim_t i) const { return i - pad_l_; } }; +namespace internal +{ +template static T constant(const T &x) { return x; } + +template +static auto replicate_construct(const T &x, std::index_sequence) +{ + return C(constant(x)...); +} + +template +static std::array replicate(const T &x, + std::index_sequence) +{ + return std::array({constant(x)...}); +} + +} // namespace internal + +template class multi_linear_sample_trait +{ + protected: + using sample_t = linear_sample_trait; + using padding_1d_t = typename sample_t::padding_t; + + struct ksize_trait; + struct stride_trait; + struct rate_trait; + + using ksize_t = std::experimental::new_type, ksize_trait>; + using stride_t = std::experimental::new_type, stride_trait>; + using rate_t = std::experimental::new_type, rate_trait>; + using padding_t = std::array; + + const std::array samples_; + + static stride_t default_stride() + { + return internal::replicate_construct( + sample_t::default_stride, std::make_index_sequence()); + } + + static rate_t default_rate() + { + return internal::replicate_construct( + sample_t::default_rate, std::make_index_sequence()); + } + + static padding_t default_padding() + { + return internal::replicate(padding_1d(sample_t::default_pad_lr), + std::make_index_sequence()); + } + + public: + static padding_1d_t padding_1d(dim_t p) { return padding_1d_t(p, p); } + + static padding_1d_t padding_1d(dim_t left, dim_t right) + { + return padding_1d_t(left, right); + } + + template static padding_t padding_simple(D... d) + { + static_assert(sizeof...(D) == r, "invalid number of arguments"); + return padding(padding_1d(static_cast(d))...); + }; + + template static padding_t padding(D... d) + { + static_assert(sizeof...(D) == r, "invalid number of arguments"); + return padding(padding_1d(static_cast(d))...); + }; + + template + static padding_t padding(const padding_1d_t &p1 /* de-ambiguity */, + const Padding1D &... p) + { + static_assert(sizeof...(Padding1D) == r - 1, + "invalid number of arguments"); + return {p1, static_cast(p)...}; + }; + + template static ksize_t ksize(D... d) + { + static_assert(sizeof...(D) == r, "invalid number of arguments"); + return ksize_t(d...); + }; + + template static stride_t stride(D... d) + { + static_assert(sizeof...(D) == r, "invalid number of arguments"); + return stride_t(d...); + }; + + template static rate_t rate(D... d) + { + static_assert(sizeof...(D) == r, "invalid number of arguments"); + return rate_t(d...); + }; + + multi_linear_sample_trait(const ksize_t &ksize) + : multi_linear_sample_trait(ksize, default_padding(), default_stride()) + { + } + + multi_linear_sample_trait(const ksize_t &ksize, const padding_t &padding) + : multi_linear_sample_trait(ksize, padding, default_stride()) + { + } + + multi_linear_sample_trait(const ksize_t &ksize, const stride_t &stride) + : multi_linear_sample_trait(ksize, default_padding(), stride) + { + } + + multi_linear_sample_trait(const ksize_t &ksize, const padding_t &padding, + const stride_t &stride) + : multi_linear_sample_trait(ksize, padding, stride, default_rate()) + { + } + + multi_linear_sample_trait(const ksize_t &ksize, const padding_t &padding, + const stride_t &stride, const rate_t &rate) + : samples_(construct(ksize, padding, stride, rate, + std::make_index_sequence())) + { + } + + template + multi_linear_sample_trait(const Sample &... sample) : samples_({sample...}) + { + } + + ksize_t get_ksize() const + { + return get_ksize(std::make_index_sequence()); + } + + shape operator()(const shape &x) const + { + return invoke(x, std::make_index_sequence()); + } + + private: + template + static std::array + construct(const ksize_t &ksize, const padding_t &padding, + const stride_t &stride, const rate_t &rate, + std::index_sequence) + { + static_assert(sizeof...(I) == r, ""); + return {sample_t(std::get(ksize.dims), std::get(stride.dims), + std::get(rate.dims), std::get(padding))...}; + } + + template + shape invoke(const shape &x, std::index_sequence) const + { + static_assert(sizeof...(I) == r, ""); + return shape(std::get(samples_)(std::get(x.dims))...); + } + + template + ksize_t get_ksize(std::index_sequence) const + { + static_assert(sizeof...(I) == r, ""); + return ksize_t(std::get(samples_).get_ksize()...); + } +}; + } // namespace nn::ops diff --git a/include/nn/bits/ops/pool.hpp b/include/nn/bits/ops/pool.hpp index 1795002..5252004 100644 --- a/include/nn/bits/ops/pool.hpp +++ b/include/nn/bits/ops/pool.hpp @@ -24,25 +24,26 @@ template <> class pool_trait static constexpr auto default_padding = padding_t(0, 0); static constexpr auto default_ksize = ksize_t(2, 2); - using sample_t = linear_sample_trait; + using dim_t = size_t; + using sample_t = linear_sample_trait; const sample_t h_sample_; const sample_t w_sample_; ksize_t get_ksize() const { - return ksize_t(h_sample_.ksize_, w_sample_.ksize_); + return ksize_t(h_sample_.get_ksize(), w_sample_.get_ksize()); } stride_t get_stride() const { - return stride_t(h_sample_.stride_, w_sample_.stride_); + return stride_t(h_sample_.get_stride(), w_sample_.get_stride()); } public: - static padding_t padding(int r, int s) { return padding_t(r, s); }; - static ksize_t ksize(int r, int s) { return ksize_t(r, s); }; - static stride_t stride(int r, int s) { return stride_t(r, s); }; + static padding_t padding(dim_t r, dim_t s) { return padding_t(r, s); }; + static ksize_t ksize(dim_t r, dim_t s) { return ksize_t(r, s); }; + static stride_t stride(dim_t r, dim_t s) { return stride_t(r, s); }; pool_trait() : pool_trait(default_ksize) {} diff --git a/tests/test_layers.cpp b/tests/test_layers.cpp index 2ea2cd8..53a1d00 100644 --- a/tests/test_layers.cpp +++ b/tests/test_layers.cpp @@ -11,16 +11,31 @@ template void test_dense_layer() l1(ref(x)); } -template void test_conv_layer() +template void test_conv_layer() { - static_assert(std::is_class::value, ""); + static_assert(std::is_class::value, ""); // static_assert(std::is_constructible::value, ""); - auto x = ttl::tensor(2, 32, 32, 32); - conv l1(conv::ksize(3, 3), 32); - l1(ref(x)); - conv l2(conv::ksize(3, 3), 32, conv::padding(1, 1)); - l2(ref(x)); + { + auto x = ttl::tensor(2, 32, 32, 32); + conv_layer l1(conv_layer::ksize(3, 3), 32); + l1(ref(x)); + conv_layer l2(conv_layer::ksize(3, 3), 32, conv_layer::padding(1, 1)); + l2(ref(x)); + } + + { + auto x = ttl::tensor(2, 224, 224, 224); + using conv_trait = nn::ops::conv_trait; + conv_layer l1( + conv_layer::ksize(7, 7), 1, + conv_trait(conv_trait::padding(conv_trait::padding_1d(3, 2), + conv_trait::padding_1d(3, 2)), + conv_trait::stride(2, 2))); + auto l = l1(ref(x)); + const auto shp = (*l).shape(); + ASSERT_EQ(shp.size(), 2 * 112 * 112); + } } template void test_pool_layer() diff --git a/tests/test_linear_sample.cpp b/tests/test_linear_sample.cpp index 3663dc1..2c60a08 100644 --- a/tests/test_linear_sample.cpp +++ b/tests/test_linear_sample.cpp @@ -56,3 +56,21 @@ void test_linear_sample_1() } TEST(linear_sample_test, test_1) { test_linear_sample_1(); } + +TEST(linear_sample_test, test_2) +{ + { + using sample_t = nn::ops::multi_linear_sample_trait<2, size_t>; + + sample_t sample(sample_t::ksize(1, 1)); + const auto y = sample(nn::shape<2>(10, 11)); + ASSERT_EQ(y, nn::shape<2>(10, 11)); + } + { + using sample_t = nn::ops::multi_linear_sample_trait<3, size_t>; + + sample_t sample(sample_t::ksize(1, 2, 3)); + const auto y = sample(nn::shape<3>(9, 8, 7)); + ASSERT_EQ(y, nn::shape<3>(9, 7, 5)); + } +}