More flex features

roboquant-ml/src/main/kotlin/org/roboquant/ml/DataFrameFeatureSet.kt

@@ -27,10 +27,23 @@ class DataFrameFeatureSet(private val historySize: Int = 1_000_000, private val
 
     private class Entry(
         val feature: Feature,
-        val data: DoubleArray,
+        val matrix: Matrix,
         val offset: Int = 0,
-        var mean: Double = Double.NaN
-    )
+        // var mean: Double = Double.NaN
+    ) {
+        fun inputData(lastRow: Int): DoubleArray {
+            val len = lastRow*feature.size
+            val result = DoubleArray(len)
+            System.arraycopy(matrix.data, 0, result, 0, len)
+            // mean = result.fillNaNMean()
+            return result
+        }
+
+        fun toDoubleVector(lastRow: Int): DoubleVector {
+            return DoubleVector.of(feature.name, inputData(lastRow))
+        }
+
+    }
 
     private val entries = mutableListOf<Entry>()
 
@@ -52,31 +65,25 @@ class DataFrameFeatureSet(private val historySize: Int = 1_000_000, private val
     fun add(vararg features: Feature, offset: Int = 0) {
         for (f in features) {
             require(f.name !in names) { "duplicate feature name ${f.name}" }
-            entries.add(Entry(f, DoubleArray(historySize), offset))
+            entries.add(Entry(f, Matrix(historySize, f.size), offset))
         }
     }
 
-    private fun Entry.inputData(size: Int): DoubleArray {
-        val result = DoubleArray(size)
-        System.arraycopy(data, 0, result, 0, size)
-        mean = result.fillNaNMean()
-        return result
-    }
 
     /**
      * Returns the training data as a [DataFrame]
      */
     fun getTrainingData(): DataFrame {
         val size = samples - maxOffset
-        val i = entries.map { DoubleVector.of(it.feature.name, it.inputData(size)) }
+        val i = entries.map {it.toDoubleVector(size) }
         @Suppress("SpreadOperator")
         return DataFrame.of(*i.toTypedArray())
     }
 
 
     private fun Entry.getRow(row: Int): DoubleVector {
         val last = samples - offset - 1
-        val doubleArr = if (row > last) doubleArrayOf(Double.NaN) else doubleArrayOf(data[row])
+        val doubleArr = if (row > last) doubleArrayOf(Double.NaN) else matrix[row]
         return DoubleVector.of(feature.name, doubleArr)
     }
 
@@ -101,11 +108,10 @@ class DataFrameFeatureSet(private val historySize: Int = 1_000_000, private val
         }
 
         for (entry in entries) {
-            var value = entry.feature.calculate(event)
+            val value = entry.feature.calculate(event)
             val idx = samples - entry.offset
             if (idx >= 0) {
-                if (value.isNaN()) value = entry.mean
-                entry.data[idx] = value
+                entry.matrix[idx] = value
             }
         }
         samples++

roboquant-ml/src/main/kotlin/org/roboquant/ml/DivFeature.kt

@@ -21,13 +21,13 @@ import org.roboquant.feeds.Event
 /**
  * Create a new feature based on the division of two other features.
  */
-class DivFeature(private val numerator: Feature, private val denominator: Feature) : Feature {
+class DivFeature(private val numerator: SingleValueFeature, private val denominator: SingleValueFeature) : SingleValueFeature() {
 
     /**
      * @see Feature.calculate
      */
-    override fun calculate(event: Event): Double {
-        return numerator.calculate(event) / denominator.calculate(event)
+    override fun calculateValue(event: Event): Double {
+        return numerator.calculateValue(event) / denominator.calculateValue(event)
     }
 
     /**
@@ -53,4 +53,4 @@ class DivFeature(private val numerator: Feature, private val denominator: Featur
  *  ```
  * @see DivFeature
  */
-operator fun Feature.div(denominator: Feature) : Feature = DivFeature(this, denominator)
+operator fun SingleValueFeature.div(denominator: SingleValueFeature) : Feature = DivFeature(this, denominator)

roboquant-ml/src/main/kotlin/org/roboquant/ml/Feature.kt

@@ -27,16 +27,40 @@ interface Feature {
     /**
      * Update the feature with a new event and return the latest value
      */
-    fun calculate(event: Event): Double
+    fun calculate(event: Event): DoubleArray
 
     /**
      * The name of this feature
      */
     val name: String
 
+    /**
+     * The size of the returned DoubleArray. This should be the same size at every event.
+     */
+    val size: Int
+
     /**
      * Reset any state in the feature
      */
     fun reset() {}
 }
 
+
+/**
+ * A feature generates a single value that is derived from a series of events
+ */
+abstract class SingleValueFeature: Feature {
+
+    override val size: Int = 1
+
+    /**
+     * Update the feature with a new event and return the latest value
+     */
+    override fun calculate(event: Event): DoubleArray {
+        return doubleArrayOf(calculateValue(event))
+    }
+
+    abstract fun calculateValue(event: Event): Double
+
+}
+

roboquant-ml/src/main/kotlin/org/roboquant/ml/HistoricPriceFeature.kt

@@ -32,14 +32,14 @@ class HistoricPriceFeature(
     private val past: Int = 1,
     private val type: String = "DEFAULT",
     override val name: String = "${asset.symbol}-HISTORIC-PRICE-$past-$type"
-) : Feature {
+) : SingleValueFeature() {
 
     private val hist = mutableListOf<Double>()
 
     /**
      * @see Feature.calculate
      */
-    override fun calculate(event: Event): Double {
+    override fun calculateValue(event: Event): Double {
         val action = event.prices[asset]
         hist.add(action?.getPrice(type) ?: Double.NaN)
         return if (hist.size > past) hist.removeFirst() else Double.NaN

roboquant-ml/src/main/kotlin/org/roboquant/ml/Matrix.kt

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2020-2023 Neural Layer
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.roboquant.ml
+
+class Matrix(private val rows: Int, private val columns: Int) {
+
+    internal val data = DoubleArray(rows * columns) {Double.NaN}
+
+    operator fun set(row: Int, value: DoubleArray) {
+        System.arraycopy(value, 0, data, row * columns, columns)
+    }
+
+    operator fun get(row: Int): DoubleArray {
+        assert(row < rows)
+        val result = DoubleArray(columns) {Double.NaN}
+        System.arraycopy(data, row*columns, result, 0, columns)
+        return result
+    }
+
+    operator fun get(start: Int, end: Int): DoubleArray {
+        assert(end < rows)
+        val len = end - start
+        val result = DoubleArray(columns * len) {Double.NaN}
+        System.arraycopy(data, start*columns, result, 0, len*columns)
+        return result
+    }
+
+}

roboquant-ml/src/main/kotlin/org/roboquant/ml/PriceFeature.kt

@@ -30,12 +30,12 @@ class PriceFeature(
     private val asset: Asset,
     private val type: String = "DEFAULT",
     override val name: String = "${asset.symbol}-PRICE-$type"
-) : Feature {
+) : SingleValueFeature() {
 
     /**
-     * @see Feature.calculate
+     * @see SingleValueFeature.calculateValue
      */
-    override fun calculate(event: Event): Double {
+    override fun calculateValue(event: Event): Double {
         val action = event.prices[asset]
         return action?.getPrice(type) ?: Double.NaN
     }

roboquant-ml/src/main/kotlin/org/roboquant/ml/ReturnsFeature.kt

@@ -16,6 +16,8 @@
 
 package org.roboquant.ml
 
+import org.roboquant.common.div
+import org.roboquant.common.minus
 import org.roboquant.feeds.Event
 
 /**
@@ -24,16 +26,18 @@ import org.roboquant.feeds.Event
 class ReturnsFeature(
     private val f: Feature,
     private val n: Int = 1,
-    private val missing: Double = Double.NaN,
+    private val missingValue: Double = Double.NaN,
     override val name: String = f.name + "-RETURNS"
-) : Feature by f {
+) : Feature {
 
-    private val history = mutableListOf<Double>()
+    private val history = mutableListOf<DoubleArray>()
+    private val missing = DoubleArray(f.size) { missingValue }
+    override val size = f.size
 
     /**
      * @see Feature.calculate
      */
-    override fun calculate(event: Event): Double {
+    override fun calculate(event: Event): DoubleArray {
         val v = f.calculate(event)
         history.add(v)
         return if (history.size > n) {
@@ -55,5 +59,5 @@ class ReturnsFeature(
 /**
  * Wrap the feature and calculate the returns
  */
-fun Feature.returns(n: Int = 1) = ReturnsFeature(this, n)
+fun SingleValueFeature.returns(n: Int = 1) = ReturnsFeature(this, n)
 

roboquant-ml/src/main/kotlin/org/roboquant/ml/TaLibFeature.kt

@@ -31,7 +31,7 @@ class TaLibFeature(
     private val asset: Asset,
     private val missing: Double = Double.NaN,
     private val block: TaLib.(prices: PriceBarSeries) -> Double
-) : Feature {
+) : SingleValueFeature() {
 
     private val taLib = TaLib()
     private val history = PriceBarSeries(1)
@@ -79,7 +79,7 @@ class TaLibFeature(
     /**
      * @see Feature.calculate
      */
-    override fun calculate(event: Event): Double {
+    override fun calculateValue(event: Event): Double {
         val action = event.prices[asset]
         if (action != null && action is PriceBar && history.add(action, event.time)) {
             try {

roboquant-ml/src/main/kotlin/org/roboquant/ml/TestFeature.kt

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2020-2023 Neural Layer
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.roboquant.ml
+
+import org.roboquant.feeds.Event
+
+/**
+ * test feature
+ */
+@Suppress("unused")
+class TestFeature(
+    private val value: DoubleArray,
+    override val name: String = "TEST-FEATURE"
+) : Feature {
+
+    override val size: Int = value.size
+
+    /**
+     * @see Feature.calculate
+     */
+    override fun calculate(event: Event): DoubleArray {
+        return  value
+    }
+
+
+}

roboquant-ml/src/main/kotlin/org/roboquant/ml/VolumeFeature.kt

@@ -25,12 +25,12 @@ import org.roboquant.feeds.Event
 class VolumeFeature(
     private val asset: Asset,
     override val name: String = "${asset.symbol}-VOLUME"
-) : Feature {
+) : SingleValueFeature() {
 
     /**
      * @see Feature.calculate
      */
-    override fun calculate(event: Event): Double {
+    override fun calculateValue(event: Event): Double {
         val action = event.prices[asset]
         return action?.volume ?: Double.NaN
     }

roboquant-ml/src/test/kotlin/org/roboquant/samples/main.kt

@@ -55,7 +55,8 @@ private fun getStrategy(asset: Asset): Strategy {
         HistoricPriceFeature(asset, 5, "CLOSE").returns(),
         HistoricPriceFeature(asset, 10, "CLOSE").returns(),
         HistoricPriceFeature(asset, 15, "CLOSE").returns(),
-        HistoricPriceFeature(asset, 30, "CLOSE").returns()
+        HistoricPriceFeature(asset, 30, "CLOSE").returns(),
+        // TestFeature(doubleArrayOf(1.0, 2.0, 3.0))
     )
 
     val s = RegressionStrategy(features, asset, 2.bips) {

roboquant-ta/src/main/kotlin/org/roboquant/ta/PriceBarSeries.kt

@@ -214,11 +214,11 @@ open class PriceBarSeries(capacity: Int) {
      * Set the capacity of the buffers to [newCapacity]. Existing stored values will be retained.
      */
     fun increaseCapacity(newCapacity: Int) {
-        openBuffer.increaeseCapacity(newCapacity)
-        highBuffer.increaeseCapacity(newCapacity)
-        lowBuffer.increaeseCapacity(newCapacity)
-        closeBuffer.increaeseCapacity(newCapacity)
-        volumeBuffer.increaeseCapacity(newCapacity)
+        openBuffer.increaseCapacity(newCapacity)
+        highBuffer.increaseCapacity(newCapacity)
+        lowBuffer.increaseCapacity(newCapacity)
+        closeBuffer.increaseCapacity(newCapacity)
+        volumeBuffer.increaseCapacity(newCapacity)
     }
 
 }

roboquant-ta/src/main/kotlin/org/roboquant/ta/RSIStrategy.kt

@@ -71,7 +71,7 @@ class RSIStrategy(
                         result.add(Signal(asset, Rating.BUY))
                 }
             } catch (ex: InsufficientData) {
-                data.increaeseCapacity(ex.minSize)
+                data.increaseCapacity(ex.minSize)
             }
         }
         return result

roboquant/src/main/kotlin/org/roboquant/common/PriceSeries.kt

@@ -104,7 +104,7 @@ open class PriceSeries(private var capacity: Int) {
     /**
      * Increase the capacity to the [newCapacity]
      */
-    fun increaeseCapacity(newCapacity: Int) {
+    fun increaseCapacity(newCapacity: Int) {
         require(newCapacity > capacity)
         val oldData = toDoubleArray()
         data = DoubleArray(newCapacity)

roboquant/src/test/kotlin/org/roboquant/strategies/utils/PriceSeriesTest.kt

@@ -55,7 +55,7 @@ internal class PriceSeriesTest {
         val series = PriceSeries(10)
         repeat(5) { series.add(1.0) }
 
-        series.increaeseCapacity(20)
+        series.increaseCapacity(20)
         assertFalse(series.isFull())
         assertEquals(5, series.size)
         assertEquals(5, series.toDoubleArray().size)