From 204b7d28e651544ad3f80e07594888272a2ef7a4 Mon Sep 17 00:00:00 2001 From: Peter Dekkers Date: Tue, 19 Dec 2023 09:21:10 +0100 Subject: [PATCH] More flex features --- .../org/roboquant/ml/DataFrameFeatureSet.kt | 36 +++++++++------- .../kotlin/org/roboquant/ml/DivFeature.kt | 8 ++-- .../main/kotlin/org/roboquant/ml/Feature.kt | 26 +++++++++++- .../org/roboquant/ml/HistoricPriceFeature.kt | 4 +- .../main/kotlin/org/roboquant/ml/Matrix.kt | 42 +++++++++++++++++++ .../kotlin/org/roboquant/ml/PriceFeature.kt | 6 +-- .../kotlin/org/roboquant/ml/ReturnsFeature.kt | 14 ++++--- .../kotlin/org/roboquant/ml/TaLibFeature.kt | 4 +- .../kotlin/org/roboquant/ml/TestFeature.kt | 40 ++++++++++++++++++ .../kotlin/org/roboquant/ml/VolumeFeature.kt | 4 +- .../test/kotlin/org/roboquant/samples/main.kt | 3 +- .../kotlin/org/roboquant/ta/PriceBarSeries.kt | 10 ++--- .../kotlin/org/roboquant/ta/RSIStrategy.kt | 2 +- .../org/roboquant/common/PriceSeries.kt | 2 +- .../strategies/utils/PriceSeriesTest.kt | 2 +- 15 files changed, 160 insertions(+), 43 deletions(-) create mode 100644 roboquant-ml/src/main/kotlin/org/roboquant/ml/Matrix.kt create mode 100644 roboquant-ml/src/main/kotlin/org/roboquant/ml/TestFeature.kt diff --git a/roboquant-ml/src/main/kotlin/org/roboquant/ml/DataFrameFeatureSet.kt b/roboquant-ml/src/main/kotlin/org/roboquant/ml/DataFrameFeatureSet.kt index ca2d028ae..a3ba9a8db 100644 --- a/roboquant-ml/src/main/kotlin/org/roboquant/ml/DataFrameFeatureSet.kt +++ b/roboquant-ml/src/main/kotlin/org/roboquant/ml/DataFrameFeatureSet.kt @@ -27,10 +27,23 @@ class DataFrameFeatureSet(private val historySize: Int = 1_000_000, private val private class Entry( val feature: Feature, - val data: DoubleArray, + val matrix: Matrix, val offset: Int = 0, - var mean: Double = Double.NaN - ) + // var mean: Double = Double.NaN + ) { + fun inputData(lastRow: Int): DoubleArray { + val len = lastRow*feature.size + val result = DoubleArray(len) + System.arraycopy(matrix.data, 0, result, 0, len) + // mean = result.fillNaNMean() + return result + } + + fun toDoubleVector(lastRow: Int): DoubleVector { + return DoubleVector.of(feature.name, inputData(lastRow)) + } + + } private val entries = mutableListOf() @@ -52,23 +65,17 @@ class DataFrameFeatureSet(private val historySize: Int = 1_000_000, private val fun add(vararg features: Feature, offset: Int = 0) { for (f in features) { require(f.name !in names) { "duplicate feature name ${f.name}" } - entries.add(Entry(f, DoubleArray(historySize), offset)) + entries.add(Entry(f, Matrix(historySize, f.size), offset)) } } - private fun Entry.inputData(size: Int): DoubleArray { - val result = DoubleArray(size) - System.arraycopy(data, 0, result, 0, size) - mean = result.fillNaNMean() - return result - } /** * Returns the training data as a [DataFrame] */ fun getTrainingData(): DataFrame { val size = samples - maxOffset - val i = entries.map { DoubleVector.of(it.feature.name, it.inputData(size)) } + val i = entries.map {it.toDoubleVector(size) } @Suppress("SpreadOperator") return DataFrame.of(*i.toTypedArray()) } @@ -76,7 +83,7 @@ class DataFrameFeatureSet(private val historySize: Int = 1_000_000, private val private fun Entry.getRow(row: Int): DoubleVector { val last = samples - offset - 1 - val doubleArr = if (row > last) doubleArrayOf(Double.NaN) else doubleArrayOf(data[row]) + val doubleArr = if (row > last) doubleArrayOf(Double.NaN) else matrix[row] return DoubleVector.of(feature.name, doubleArr) } @@ -101,11 +108,10 @@ class DataFrameFeatureSet(private val historySize: Int = 1_000_000, private val } for (entry in entries) { - var value = entry.feature.calculate(event) + val value = entry.feature.calculate(event) val idx = samples - entry.offset if (idx >= 0) { - if (value.isNaN()) value = entry.mean - entry.data[idx] = value + entry.matrix[idx] = value } } samples++ diff --git a/roboquant-ml/src/main/kotlin/org/roboquant/ml/DivFeature.kt b/roboquant-ml/src/main/kotlin/org/roboquant/ml/DivFeature.kt index 98a697913..21dca1a94 100644 --- a/roboquant-ml/src/main/kotlin/org/roboquant/ml/DivFeature.kt +++ b/roboquant-ml/src/main/kotlin/org/roboquant/ml/DivFeature.kt @@ -21,13 +21,13 @@ import org.roboquant.feeds.Event /** * Create a new feature based on the division of two other features. */ -class DivFeature(private val numerator: Feature, private val denominator: Feature) : Feature { +class DivFeature(private val numerator: SingleValueFeature, private val denominator: SingleValueFeature) : SingleValueFeature() { /** * @see Feature.calculate */ - override fun calculate(event: Event): Double { - return numerator.calculate(event) / denominator.calculate(event) + override fun calculateValue(event: Event): Double { + return numerator.calculateValue(event) / denominator.calculateValue(event) } /** @@ -53,4 +53,4 @@ class DivFeature(private val numerator: Feature, private val denominator: Featur * ``` * @see DivFeature */ -operator fun Feature.div(denominator: Feature) : Feature = DivFeature(this, denominator) +operator fun SingleValueFeature.div(denominator: SingleValueFeature) : Feature = DivFeature(this, denominator) diff --git a/roboquant-ml/src/main/kotlin/org/roboquant/ml/Feature.kt b/roboquant-ml/src/main/kotlin/org/roboquant/ml/Feature.kt index 1e17affb2..7fa1649e3 100644 --- a/roboquant-ml/src/main/kotlin/org/roboquant/ml/Feature.kt +++ b/roboquant-ml/src/main/kotlin/org/roboquant/ml/Feature.kt @@ -27,16 +27,40 @@ interface Feature { /** * Update the feature with a new event and return the latest value */ - fun calculate(event: Event): Double + fun calculate(event: Event): DoubleArray /** * The name of this feature */ val name: String + /** + * The size of the returned DoubleArray. This should be the same size at every event. + */ + val size: Int + /** * Reset any state in the feature */ fun reset() {} } + +/** + * A feature generates a single value that is derived from a series of events + */ +abstract class SingleValueFeature: Feature { + + override val size: Int = 1 + + /** + * Update the feature with a new event and return the latest value + */ + override fun calculate(event: Event): DoubleArray { + return doubleArrayOf(calculateValue(event)) + } + + abstract fun calculateValue(event: Event): Double + +} + diff --git a/roboquant-ml/src/main/kotlin/org/roboquant/ml/HistoricPriceFeature.kt b/roboquant-ml/src/main/kotlin/org/roboquant/ml/HistoricPriceFeature.kt index ae884f83f..565941e98 100644 --- a/roboquant-ml/src/main/kotlin/org/roboquant/ml/HistoricPriceFeature.kt +++ b/roboquant-ml/src/main/kotlin/org/roboquant/ml/HistoricPriceFeature.kt @@ -32,14 +32,14 @@ class HistoricPriceFeature( private val past: Int = 1, private val type: String = "DEFAULT", override val name: String = "${asset.symbol}-HISTORIC-PRICE-$past-$type" -) : Feature { +) : SingleValueFeature() { private val hist = mutableListOf() /** * @see Feature.calculate */ - override fun calculate(event: Event): Double { + override fun calculateValue(event: Event): Double { val action = event.prices[asset] hist.add(action?.getPrice(type) ?: Double.NaN) return if (hist.size > past) hist.removeFirst() else Double.NaN diff --git a/roboquant-ml/src/main/kotlin/org/roboquant/ml/Matrix.kt b/roboquant-ml/src/main/kotlin/org/roboquant/ml/Matrix.kt new file mode 100644 index 000000000..519bc591e --- /dev/null +++ b/roboquant-ml/src/main/kotlin/org/roboquant/ml/Matrix.kt @@ -0,0 +1,42 @@ +/* + * Copyright 2020-2023 Neural Layer + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.roboquant.ml + +class Matrix(private val rows: Int, private val columns: Int) { + + internal val data = DoubleArray(rows * columns) {Double.NaN} + + operator fun set(row: Int, value: DoubleArray) { + System.arraycopy(value, 0, data, row * columns, columns) + } + + operator fun get(row: Int): DoubleArray { + assert(row < rows) + val result = DoubleArray(columns) {Double.NaN} + System.arraycopy(data, row*columns, result, 0, columns) + return result + } + + operator fun get(start: Int, end: Int): DoubleArray { + assert(end < rows) + val len = end - start + val result = DoubleArray(columns * len) {Double.NaN} + System.arraycopy(data, start*columns, result, 0, len*columns) + return result + } + +} diff --git a/roboquant-ml/src/main/kotlin/org/roboquant/ml/PriceFeature.kt b/roboquant-ml/src/main/kotlin/org/roboquant/ml/PriceFeature.kt index a0e507bec..10d9e6d0a 100644 --- a/roboquant-ml/src/main/kotlin/org/roboquant/ml/PriceFeature.kt +++ b/roboquant-ml/src/main/kotlin/org/roboquant/ml/PriceFeature.kt @@ -30,12 +30,12 @@ class PriceFeature( private val asset: Asset, private val type: String = "DEFAULT", override val name: String = "${asset.symbol}-PRICE-$type" -) : Feature { +) : SingleValueFeature() { /** - * @see Feature.calculate + * @see SingleValueFeature.calculateValue */ - override fun calculate(event: Event): Double { + override fun calculateValue(event: Event): Double { val action = event.prices[asset] return action?.getPrice(type) ?: Double.NaN } diff --git a/roboquant-ml/src/main/kotlin/org/roboquant/ml/ReturnsFeature.kt b/roboquant-ml/src/main/kotlin/org/roboquant/ml/ReturnsFeature.kt index 1d3d91a63..7ce7127e6 100644 --- a/roboquant-ml/src/main/kotlin/org/roboquant/ml/ReturnsFeature.kt +++ b/roboquant-ml/src/main/kotlin/org/roboquant/ml/ReturnsFeature.kt @@ -16,6 +16,8 @@ package org.roboquant.ml +import org.roboquant.common.div +import org.roboquant.common.minus import org.roboquant.feeds.Event /** @@ -24,16 +26,18 @@ import org.roboquant.feeds.Event class ReturnsFeature( private val f: Feature, private val n: Int = 1, - private val missing: Double = Double.NaN, + private val missingValue: Double = Double.NaN, override val name: String = f.name + "-RETURNS" -) : Feature by f { +) : Feature { - private val history = mutableListOf() + private val history = mutableListOf() + private val missing = DoubleArray(f.size) { missingValue } + override val size = f.size /** * @see Feature.calculate */ - override fun calculate(event: Event): Double { + override fun calculate(event: Event): DoubleArray { val v = f.calculate(event) history.add(v) return if (history.size > n) { @@ -55,5 +59,5 @@ class ReturnsFeature( /** * Wrap the feature and calculate the returns */ -fun Feature.returns(n: Int = 1) = ReturnsFeature(this, n) +fun SingleValueFeature.returns(n: Int = 1) = ReturnsFeature(this, n) diff --git a/roboquant-ml/src/main/kotlin/org/roboquant/ml/TaLibFeature.kt b/roboquant-ml/src/main/kotlin/org/roboquant/ml/TaLibFeature.kt index 022cc5cac..4cb74807b 100644 --- a/roboquant-ml/src/main/kotlin/org/roboquant/ml/TaLibFeature.kt +++ b/roboquant-ml/src/main/kotlin/org/roboquant/ml/TaLibFeature.kt @@ -31,7 +31,7 @@ class TaLibFeature( private val asset: Asset, private val missing: Double = Double.NaN, private val block: TaLib.(prices: PriceBarSeries) -> Double -) : Feature { +) : SingleValueFeature() { private val taLib = TaLib() private val history = PriceBarSeries(1) @@ -79,7 +79,7 @@ class TaLibFeature( /** * @see Feature.calculate */ - override fun calculate(event: Event): Double { + override fun calculateValue(event: Event): Double { val action = event.prices[asset] if (action != null && action is PriceBar && history.add(action, event.time)) { try { diff --git a/roboquant-ml/src/main/kotlin/org/roboquant/ml/TestFeature.kt b/roboquant-ml/src/main/kotlin/org/roboquant/ml/TestFeature.kt new file mode 100644 index 000000000..d22697eda --- /dev/null +++ b/roboquant-ml/src/main/kotlin/org/roboquant/ml/TestFeature.kt @@ -0,0 +1,40 @@ +/* + * Copyright 2020-2023 Neural Layer + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.roboquant.ml + +import org.roboquant.feeds.Event + +/** + * test feature + */ +@Suppress("unused") +class TestFeature( + private val value: DoubleArray, + override val name: String = "TEST-FEATURE" +) : Feature { + + override val size: Int = value.size + + /** + * @see Feature.calculate + */ + override fun calculate(event: Event): DoubleArray { + return value + } + + +} diff --git a/roboquant-ml/src/main/kotlin/org/roboquant/ml/VolumeFeature.kt b/roboquant-ml/src/main/kotlin/org/roboquant/ml/VolumeFeature.kt index b7263941d..15546f7bf 100644 --- a/roboquant-ml/src/main/kotlin/org/roboquant/ml/VolumeFeature.kt +++ b/roboquant-ml/src/main/kotlin/org/roboquant/ml/VolumeFeature.kt @@ -25,12 +25,12 @@ import org.roboquant.feeds.Event class VolumeFeature( private val asset: Asset, override val name: String = "${asset.symbol}-VOLUME" -) : Feature { +) : SingleValueFeature() { /** * @see Feature.calculate */ - override fun calculate(event: Event): Double { + override fun calculateValue(event: Event): Double { val action = event.prices[asset] return action?.volume ?: Double.NaN } diff --git a/roboquant-ml/src/test/kotlin/org/roboquant/samples/main.kt b/roboquant-ml/src/test/kotlin/org/roboquant/samples/main.kt index 275827217..a4be6c866 100644 --- a/roboquant-ml/src/test/kotlin/org/roboquant/samples/main.kt +++ b/roboquant-ml/src/test/kotlin/org/roboquant/samples/main.kt @@ -55,7 +55,8 @@ private fun getStrategy(asset: Asset): Strategy { HistoricPriceFeature(asset, 5, "CLOSE").returns(), HistoricPriceFeature(asset, 10, "CLOSE").returns(), HistoricPriceFeature(asset, 15, "CLOSE").returns(), - HistoricPriceFeature(asset, 30, "CLOSE").returns() + HistoricPriceFeature(asset, 30, "CLOSE").returns(), + // TestFeature(doubleArrayOf(1.0, 2.0, 3.0)) ) val s = RegressionStrategy(features, asset, 2.bips) { diff --git a/roboquant-ta/src/main/kotlin/org/roboquant/ta/PriceBarSeries.kt b/roboquant-ta/src/main/kotlin/org/roboquant/ta/PriceBarSeries.kt index be92f4fc8..f5bfac698 100644 --- a/roboquant-ta/src/main/kotlin/org/roboquant/ta/PriceBarSeries.kt +++ b/roboquant-ta/src/main/kotlin/org/roboquant/ta/PriceBarSeries.kt @@ -214,11 +214,11 @@ open class PriceBarSeries(capacity: Int) { * Set the capacity of the buffers to [newCapacity]. Existing stored values will be retained. */ fun increaseCapacity(newCapacity: Int) { - openBuffer.increaeseCapacity(newCapacity) - highBuffer.increaeseCapacity(newCapacity) - lowBuffer.increaeseCapacity(newCapacity) - closeBuffer.increaeseCapacity(newCapacity) - volumeBuffer.increaeseCapacity(newCapacity) + openBuffer.increaseCapacity(newCapacity) + highBuffer.increaseCapacity(newCapacity) + lowBuffer.increaseCapacity(newCapacity) + closeBuffer.increaseCapacity(newCapacity) + volumeBuffer.increaseCapacity(newCapacity) } } diff --git a/roboquant-ta/src/main/kotlin/org/roboquant/ta/RSIStrategy.kt b/roboquant-ta/src/main/kotlin/org/roboquant/ta/RSIStrategy.kt index 490d43571..0934a83b8 100644 --- a/roboquant-ta/src/main/kotlin/org/roboquant/ta/RSIStrategy.kt +++ b/roboquant-ta/src/main/kotlin/org/roboquant/ta/RSIStrategy.kt @@ -71,7 +71,7 @@ class RSIStrategy( result.add(Signal(asset, Rating.BUY)) } } catch (ex: InsufficientData) { - data.increaeseCapacity(ex.minSize) + data.increaseCapacity(ex.minSize) } } return result diff --git a/roboquant/src/main/kotlin/org/roboquant/common/PriceSeries.kt b/roboquant/src/main/kotlin/org/roboquant/common/PriceSeries.kt index 61c33b4c3..780a904c3 100644 --- a/roboquant/src/main/kotlin/org/roboquant/common/PriceSeries.kt +++ b/roboquant/src/main/kotlin/org/roboquant/common/PriceSeries.kt @@ -104,7 +104,7 @@ open class PriceSeries(private var capacity: Int) { /** * Increase the capacity to the [newCapacity] */ - fun increaeseCapacity(newCapacity: Int) { + fun increaseCapacity(newCapacity: Int) { require(newCapacity > capacity) val oldData = toDoubleArray() data = DoubleArray(newCapacity) diff --git a/roboquant/src/test/kotlin/org/roboquant/strategies/utils/PriceSeriesTest.kt b/roboquant/src/test/kotlin/org/roboquant/strategies/utils/PriceSeriesTest.kt index d0211d679..85e659a6c 100644 --- a/roboquant/src/test/kotlin/org/roboquant/strategies/utils/PriceSeriesTest.kt +++ b/roboquant/src/test/kotlin/org/roboquant/strategies/utils/PriceSeriesTest.kt @@ -55,7 +55,7 @@ internal class PriceSeriesTest { val series = PriceSeries(10) repeat(5) { series.add(1.0) } - series.increaeseCapacity(20) + series.increaseCapacity(20) assertFalse(series.isFull()) assertEquals(5, series.size) assertEquals(5, series.toDoubleArray().size)