diff --git a/pom.xml b/pom.xml index f0e5158a..cf69978d 100644 --- a/pom.xml +++ b/pom.xml @@ -56,7 +56,6 @@ roboquant-avro roboquant-charts roboquant-ta - roboquant-ml roboquant-jupyter roboquant-alphavantage roboquant-alpaca diff --git a/roboquant-avro/src/main/kotlin/org/roboquant/avro/AvroFeed.kt b/roboquant-avro/src/main/kotlin/org/roboquant/avro/AvroFeed.kt index dc49c09e..95ff4f7e 100644 --- a/roboquant-avro/src/main/kotlin/org/roboquant/avro/AvroFeed.kt +++ b/roboquant-avro/src/main/kotlin/org/roboquant/avro/AvroFeed.kt @@ -16,107 +16,85 @@ package org.roboquant.avro +import kotlinx.coroutines.channels.ClosedReceiveChannelException +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import org.apache.avro.Schema +import org.apache.avro.file.CodecFactory +import org.apache.avro.file.DataFileConstants import org.apache.avro.file.DataFileReader +import org.apache.avro.file.DataFileWriter +import org.apache.avro.generic.GenericData import org.apache.avro.generic.GenericDatumReader +import org.apache.avro.generic.GenericDatumWriter import org.apache.avro.generic.GenericRecord +import org.apache.avro.io.DatumWriter import org.apache.avro.util.Utf8 -import org.roboquant.common.* +import org.roboquant.common.Asset +import org.roboquant.common.Logging +import org.roboquant.common.Timeframe +import org.roboquant.common.compareTo import org.roboquant.feeds.* -import java.io.InputStream -import java.net.URL import java.nio.file.Files import java.nio.file.Path -import java.nio.file.Paths -import java.nio.file.StandardCopyOption import java.time.Instant import java.util.* -import kotlin.io.path.isRegularFile +internal const val SCHEMA = """{ + "namespace": "org.roboquant.avro.schema", + "type": "record", + "name": "PriceItemV2", + "fields": [ + {"name": "timestamp_nanos", "type" : "long"}, + {"name": "symbol", "type": "string"}, + {"name": "type", "type": { "type": "enum", "name": "item_type", "symbols" : ["BAR", "TRADE", "QUOTE", "BOOK"]}}, + {"name": "values", "type": {"type": "array", "items" : "double"}}, + {"name": "other", "type": ["null", "string"], "default": null} + ] + }""" /** * Read price data from a single file in Avro format. This feed loads data lazy and disposes of it afterwards, so * memory footprint is low. Compared to CSV files, Avro files are parsed more efficient, making it a good fit for large * back tests. Additionally, an Avro file can be compressed, reducing the overall disk space required. * - * When the feed is instantiated, it will create an internal index for faster random access. Please note that - * currently the internal resolution is milliseconds. + * The internal resolution is nanoseconds and stored as a single Long value * * @property path the path where the Avro file can be found + * @property template template to use to convert the stored symbols into assets * * @constructor Create new Avro Feed */ -class AvroFeed(private val path: Path, useCache: Boolean = false) : AssetFeed { +class AvroFeed(private val path: Path, private val template: Asset = Asset("TEMPLATE")) : Feed { /** * Instantiate an Avro Feed based on the Avro file at [path] */ constructor(path: String) : this(Path.of(path)) - /** - * Contains mapping of a serialized Asset string to an Asset - */ - private val assetLookup: Map + private val logger = Logging.getLogger(AvroFeed::class) - /** - * MetadataProvider that holds time/position for quicker access rows - * in Avro file. - */ - private val index: List> - - /** - * @see Feed.timeframe - */ - override val timeframe: Timeframe + private val index by lazy { createIndex() } - /** - * Get available assets. - */ - override val assets: SortedSet - get() = assetLookup.values.toSortedSet() + override val timeframe: Timeframe by lazy { calcTimeframe() } init { - assert(path.isRegularFile()) { "$path is not a file" } - val metadataProvider = MetadataProvider(path) - - val metadata = metadataProvider.build(useCache) - this.index = metadata.index - timeframe = metadata.timeframe - assetLookup = metadata.assets - - logger.info { "loaded feed with timeframe=$timeframe" } + logger.info { "New AvroFeed path=$path exist=${exists()}" } } - private fun position(r: DataFileReader, time: Instant) { - val idx = index.binarySearch { it.first.compareTo(time) } - when { - idx > 0 -> r.seek(index[idx - 1].second) - idx < -1 -> r.seek(index[-idx - 2].second) - } - } + fun exists(): Boolean = Files.exists(path) private fun getReader(): DataFileReader { return DataFileReader(path.toFile(), GenericDatumReader()) } - /** - * Convert a generic Avro record to a [PriceItem] - */ - private fun recToPriceAction(rec: GenericRecord, serializer: PriceActionSerializer): PriceItem { - val assetStr = rec.get(1).toString() - val asset = assetLookup.getValue(assetStr) - val actionType = rec.get(2) as Int - - @Suppress("UNCHECKED_CAST") - val values = rec.get(3) as List - - return if (rec.hasField("other")) { - val other = rec.get("other") as Utf8? - serializer.deserialize(asset, actionType, values, other?.toString()) - } else { - serializer.deserialize(asset, actionType, values, null) - } + private fun ofEpochNano(value: Long): Instant { + return if (value >= 0L) + Instant.ofEpochSecond(value / 1_000_000_000L, value % 1_000_000_000L) + else + Instant.ofEpochSecond(value / 1_000_000_000L, -value % 1_000_000_000L) } /** @@ -128,116 +106,160 @@ class AvroFeed(private val path: Path, useCache: Boolean = false) : AssetFeed { override suspend fun play(channel: EventChannel) { val timeframe = channel.timeframe var last = Instant.MIN - var actions = ArrayList() + var items = ArrayList() val serializer = PriceActionSerializer() - + val cache = mutableMapOf() getReader().use { - position(it, timeframe.start) - + if (timeframe.isFinite()) position(it, timeframe.start) while (it.hasNext()) { val rec = it.next() // Optimize unnecessary parsing of the whole record - val now = Instant.ofEpochMilli(rec[0] as Long) + val now = ofEpochNano(rec[0] as Long) if (now < timeframe) continue if (now != last) { - channel.sendNotEmpty(Event(last, actions)) + channel.sendNotEmpty(Event(last, items)) last = now - actions = ArrayList(actions.size) + items = ArrayList(items.size) } if (now > timeframe) break - val action = recToPriceAction(rec, serializer) - actions.add(action) + // Parse the remaining attributes + val symbol = rec.get(1).toString() + val asset = cache.getOrPut(symbol) { template.copy(symbol = symbol) } + val priceItemType = PriceItemType.valueOf(rec.get(2).toString()) + + @Suppress("UNCHECKED_CAST") + val values = rec.get(3) as List + val other = rec.get("other") as Utf8? + val item = serializer.deserialize(asset, priceItemType, values, other?.toString()) + items.add(item) + } + channel.sendNotEmpty(Event(last, items)) + } + } + + private fun position(r: DataFileReader, time: Instant) { + val key = index.floorKey(time) + if (key != null) r.seek(index.getValue(key)) + } + + private fun createIndex() : TreeMap { + val index = TreeMap() + getReader().use { + while (it.hasNext()) { + val position = it.tell() + val t = ofEpochNano(it.next().get(0) as Long) + it.seek(position) + if (it.hasNext()) { + index.putIfAbsent(t,position) + it.nextBlock() + } + } + } + return index + } + + private fun calcTimeframe() : Timeframe { + if (index.isEmpty()) return Timeframe.EMPTY + val start = index.firstKey() + getReader().use { + position(it, index.lastKey()) + var timestamp = index.lastKey().toEpochNano() + while (it.hasNext()) { + timestamp = it.next().get(0) as Long } - channel.sendNotEmpty(Event(last, actions)) + return Timeframe(start, ofEpochNano(timestamp), true) } } + private fun Instant.toEpochNano(): Long { + var currentTimeNano = epochSecond * 1_000_000_000L + currentTimeNano += if (currentTimeNano > 0) nano else -nano + return currentTimeNano + } + /** - * Standard set of Avro feeds that come with roboquant and will be downloaded the first time when invoked. They are - * stored at /.roboquant and reused from there later on. + * Record the price-actions in a [feed] and store them in an Avro file that can be later used as input for + * an AvroFeed. The provided [feed] needs to implement the [AssetFeed] interface. + * + * [compress] can be enabled, which results in a smaller file. The `snappy` compression codec is used, that + * achieves decent compression ratio while using limited CPU usage. + * + * Additionally, you can filter on a [timeframe]. Default is to apply no filtering. */ - companion object { - - internal val logger = Logging.getLogger(AvroFeed::class) - private const val SP500FILE = "sp500_pricebar_v6.0.avro" - private const val SP500QUOTEFILE = "sp500_pricequote_v5.0.avro" - private const val FOREXFILE = "forex_pricebar_v5.1.avro" - - /** - * Get an AvroFeed containing end-of-day [PriceBar] data for the companies listed in the S&P 500. This feed - * contains a few years of public data. - * - * Please note that not all US exchanges are included, so the prices are not 100% accurate. - */ - fun sp500(): AvroFeed { - val path = download(SP500FILE) - return AvroFeed(path) + @Suppress("LongParameterList") + fun record( + feed: Feed, + compress: Boolean = true, + timeframe: Timeframe = Timeframe.INFINITE, + append: Boolean = false, + syncInterval: Int = DataFileConstants.DEFAULT_SYNC_INTERVAL + ) = runBlocking { + + val channel = EventChannel(timeframe = timeframe) + val schema = Schema.Parser().parse(SCHEMA) + val datumWriter: DatumWriter = GenericDatumWriter(schema) + val dataFileWriter = DataFileWriter(datumWriter) + val file = path.toFile() + + if (append) { + require(exists()) {"File $file doesn't exist yet, cannot append"} + dataFileWriter.appendTo(file) + } else { + if (compress) dataFileWriter.setCodec(CodecFactory.snappyCodec()) + dataFileWriter.setSyncInterval(syncInterval) + dataFileWriter.create(schema, file) } - /** - * Get an AvroFeed containing [PriceQuote] data for the companies listed in the S&P 500. This feed contains - * a few minutes of public data. - * - * Please note that not all US exchanges are included, so the prices are not 100% accurate. - */ - fun sp500Quotes(): AvroFeed { - val path = download(SP500QUOTEFILE) - return AvroFeed(path) + val job = launch { + feed.play(channel) + channel.close() } - /** - * Get an AvroFeed containing 1 minute [PriceBar] data for an EUR/USD currency pair. - */ - fun forex(): AvroFeed { - val path = download(FOREXFILE) - return AvroFeed(path) - } + val arraySchema = Schema.createArray(Schema.create(Schema.Type.DOUBLE)) + val enumSchema = Schema.createArray(Schema.create(Schema.Type.STRING)) + try { + val record = GenericData.Record(schema) + val serializer = PriceActionSerializer() + + while (true) { + val event = channel.receive() + val now = event.time.toEpochNano() + + for (action in event.items.filterIsInstance()) { + + val asset = action.asset + record.put(0, now) + record.put(1, asset.symbol) - /** - * Download a file from GitHub if now yet present on the local file system. - */ - private fun download(fileName: String): Path { - val path: Path = Paths.get(Config.home.toString(), fileName) - if (Files.notExists(path)) { - val url = "https://roboquant-public.s3.eu-west-1.amazonaws.com/avro/$fileName" - // val url = "https://github.com/neurallayer/roboquant-data/blob/main/avro/$fileName?raw=true" - logger.info("Downloading data from $url...") - val website = URL(url) - website.openStream().use { inputStream: InputStream -> - Files.copy( - inputStream, path, StandardCopyOption.REPLACE_EXISTING - ) + val serialization = serializer.serialize(action) + val t = GenericData.EnumSymbol(enumSchema, serialization.type) + record.put(2, t) + + val arr = GenericData.Array(serialization.values.size, arraySchema) + arr.addAll(serialization.values) + record.put(3, arr) + + record.put(4, serialization.other) + dataFileWriter.append(record) } - require(Files.exists(path)) + } - return path - } - /** - * Record the price-actions in a [feed] and store them in an Avro [fileName] that can be later used as input for - * an AvroFeed. The provided [feed] needs to implement the [AssetFeed] interface. - * - * [compression] can be enabled, which results in a smaller file. The `snappy` compression codec is used, that - * achieves decent compression ratio while using limited CPU usage. - * - * Additionally, you can filter on a [timeframe] and [assetFilter]. Default is to apply no filtering. - */ - @Suppress("LongParameterList") - fun record( - feed: Feed, - fileName: String, - compression: Boolean = true, - timeframe: Timeframe = Timeframe.INFINITE, - append: Boolean = false, - assetFilter: AssetFilter = AssetFilter.all() - ) { - recordAvro(feed, fileName, compression, timeframe, append, assetFilter) + } catch (_: ClosedReceiveChannelException) { + // On purpose left empty, expected exception + } finally { + channel.close() + if (job.isActive) job.cancel() + dataFileWriter.sync() + dataFileWriter.close() } } + + } diff --git a/roboquant-avro/src/main/kotlin/org/roboquant/avro/AvroFeed2.kt b/roboquant-avro/src/main/kotlin/org/roboquant/avro/AvroFeed2.kt deleted file mode 100644 index 1dfbbc1a..00000000 --- a/roboquant-avro/src/main/kotlin/org/roboquant/avro/AvroFeed2.kt +++ /dev/null @@ -1,263 +0,0 @@ -/* - * Copyright 2020-2024 Neural Layer - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.roboquant.avro - -import kotlinx.coroutines.channels.ClosedReceiveChannelException -import kotlinx.coroutines.launch -import kotlinx.coroutines.runBlocking -import org.apache.avro.Schema -import org.apache.avro.file.CodecFactory -import org.apache.avro.file.DataFileConstants -import org.apache.avro.file.DataFileReader -import org.apache.avro.file.DataFileWriter -import org.apache.avro.generic.GenericData -import org.apache.avro.generic.GenericDatumReader -import org.apache.avro.generic.GenericDatumWriter -import org.apache.avro.generic.GenericRecord -import org.apache.avro.io.DatumWriter -import org.apache.avro.util.Utf8 -import org.roboquant.common.Asset -import org.roboquant.common.Logging -import org.roboquant.common.Timeframe -import org.roboquant.common.compareTo -import org.roboquant.feeds.* -import java.nio.file.Files -import java.nio.file.Path -import java.time.Instant -import java.util.* - - -/** - * Read price data from a single file in Avro format. This feed loads data lazy and disposes of it afterwards, so - * memory footprint is low. Compared to CSV files, Avro files are parsed more efficient, making it a good fit for large - * back tests. Additionally, an Avro file can be compressed, reducing the overall disk space required. - * - * The internal resolution is nanoseconds and stored as a single Long value - * - * @property path the path where the Avro file can be found - * @property template template to use to convert the stored symbols into assets - * - * @constructor Create new Avro Feed - */ -class AvroFeed2(private val path: Path, private val template: Asset = Asset("TEMPLATE")) : Feed { - - /** - * Instantiate an Avro Feed based on the Avro file at [path] - */ - constructor(path: String) : this(Path.of(path)) - - private val logger = Logging.getLogger(AvroFeed2::class) - - private val index by lazy { createIndex() } - - override val timeframe: Timeframe by lazy { calcTimeframe() } - - - init { - logger.info { "New AvroFeed path=$path exist=${exists()}" } - } - - - fun exists(): Boolean = Files.exists(path) - - private fun getReader(): DataFileReader { - return DataFileReader(path.toFile(), GenericDatumReader()) - } - - private fun ofEpochNano(value: Long): Instant { - return if (value >= 0L) - Instant.ofEpochSecond(value / 1_000_000_000L, value % 1_000_000_000L) - else - Instant.ofEpochSecond(value / 1_000_000_000L, -value % 1_000_000_000L) - } - - /** - * (Re)play the events of the feed using the provided [EventChannel] - * - * @param channel - * @return - */ - override suspend fun play(channel: EventChannel) { - val timeframe = channel.timeframe - var last = Instant.MIN - var items = ArrayList() - val serializer = PriceActionSerializer() - val cache = mutableMapOf() - getReader().use { - if (timeframe.isFinite()) position(it, timeframe.start) - while (it.hasNext()) { - val rec = it.next() - - // Optimize unnecessary parsing of the whole record - val now = ofEpochNano(rec[0] as Long) - if (now < timeframe) continue - - if (now != last) { - channel.sendNotEmpty(Event(last, items)) - last = now - items = ArrayList(items.size) - } - - if (now > timeframe) break - - // Parse the remaining attributes - val symbol = rec.get(1).toString() - val asset = cache.getOrPut(symbol) { template.copy(symbol = symbol) } - val actionType = rec.get(2) as Int - - @Suppress("UNCHECKED_CAST") - val values = rec.get(3) as List - val other = rec.get("other") as Utf8? - val item = serializer.deserialize(asset, actionType, values, other?.toString()) - items.add(item) - } - channel.sendNotEmpty(Event(last, items)) - } - } - - private fun position(r: DataFileReader, time: Instant) { - val key = index.floorKey(time) - if (key != null) r.seek(index.getValue(key)) - } - - private fun createIndex() : TreeMap { - val index = TreeMap() - getReader().use { - while (it.hasNext()) { - val position = it.tell() - val t = ofEpochNano(it.next().get(0) as Long) - it.seek(position) - if (it.hasNext()) { - index.putIfAbsent(t,position) - it.nextBlock() - } - } - } - return index - } - - private fun calcTimeframe() : Timeframe { - if (index.isEmpty()) return Timeframe.EMPTY - val start = index.firstKey() - getReader().use { - position(it, index.lastKey()) - var timestamp = index.lastKey().toEpochNano() - while (it.hasNext()) { - timestamp = it.next().get(0) as Long - } - return Timeframe(start, ofEpochNano(timestamp), true) - } - } - - private fun Instant.toEpochNano(): Long { - var currentTimeNano = epochSecond * 1_000_000_000L - currentTimeNano += if (currentTimeNano > 0) nano else -nano - return currentTimeNano - } - - /** - * Record the price-actions in a [feed] and store them in an Avro file that can be later used as input for - * an AvroFeed. The provided [feed] needs to implement the [AssetFeed] interface. - * - * [compression] can be enabled, which results in a smaller file. The `snappy` compression codec is used, that - * achieves decent compression ratio while using limited CPU usage. - * - * Additionally, you can filter on a [timeframe]. Default is to apply no filtering. - */ - @Suppress("LongParameterList") - fun record( - feed: Feed, - compression: Boolean = true, - timeframe: Timeframe = Timeframe.INFINITE, - append: Boolean = false, - syncInterval: Int = DataFileConstants.DEFAULT_SYNC_INTERVAL - ) = runBlocking { - val schemaDef = """{ - "namespace": "org.roboquant.avro.schema", - "type": "record", - "name": "PriceItemV2", - "fields": [ - {"name": "timestamp_ns", "type": "long"}, - {"name": "symbol", "type": "string"}, - {"name": "type", "type": "int"}, - {"name": "values", "type": {"type": "array", "items" : "double"}}, - {"name": "other", "type": ["null", "string"], "default": null} - ] - }""" - - val channel = EventChannel(timeframe = timeframe) - val schema = Schema.Parser().parse(schemaDef) - val datumWriter: DatumWriter = GenericDatumWriter(schema) - val dataFileWriter = DataFileWriter(datumWriter) - val file = path.toFile() - - if (append) { - require(exists()) {"File $file doesn't exist yet, cannot append"} - dataFileWriter.appendTo(file) - } else { - if (compression) dataFileWriter.setCodec(CodecFactory.snappyCodec()) - dataFileWriter.setSyncInterval(syncInterval) - dataFileWriter.create(schema, file) - } - - val job = launch { - feed.play(channel) - channel.close() - } - - val arraySchema = Schema.createArray(Schema.create(Schema.Type.DOUBLE)) - try { - val record = GenericData.Record(schema) - val serializer = PriceActionSerializer() - - while (true) { - val event = channel.receive() - val now = event.time.toEpochNano() - - for (action in event.items.filterIsInstance()) { - - val asset = action.asset - record.put(0, now) - record.put(1, asset.symbol) - - val serialization = serializer.serialize(action) - record.put(2, serialization.type) - - val arr = GenericData.Array(serialization.values.size, arraySchema) - arr.addAll(serialization.values) - record.put(3, arr) - - record.put(4, serialization.other) - dataFileWriter.append(record) - } - - } - - } catch (_: ClosedReceiveChannelException) { - // On purpose left empty, expected exception - } finally { - channel.close() - if (job.isActive) job.cancel() - dataFileWriter.sync() - dataFileWriter.close() - } - } - - - -} - diff --git a/roboquant-avro/src/main/kotlin/org/roboquant/avro/MetadataProvider.kt b/roboquant-avro/src/main/kotlin/org/roboquant/avro/MetadataProvider.kt deleted file mode 100644 index d7754dab..00000000 --- a/roboquant-avro/src/main/kotlin/org/roboquant/avro/MetadataProvider.kt +++ /dev/null @@ -1,205 +0,0 @@ -/* - * Copyright 2020-2024 Neural Layer - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.roboquant.avro - -import org.apache.avro.file.DataFileReader -import org.apache.avro.generic.GenericDatumReader -import org.apache.avro.generic.GenericRecord -import org.roboquant.common.Asset -import org.roboquant.common.RoboquantException -import org.roboquant.common.Timeframe -import org.roboquant.feeds.util.AssetSerializer.deserialize -import java.io.* -import java.nio.file.Path -import java.security.MessageDigest -import java.time.Instant -import kotlin.io.path.pathString - -/** - * MetadataProvider of the feed that optionally can be loaded from disk directly to memory for speedy startup times - */ -internal class MetadataProvider(private val avroFile: Path) { - - private val cacheFile = File(avroFile.pathString + CACHE_SUFFIX) - - internal data class Metadata( - val index: List>, - val assets: Map, - val timeframe: Timeframe - ) - - private fun toSerializable(data: Metadata): Triple>, List, Timeframe> { - return Triple( - data.index, - data.assets.keys.toList(), - data.timeframe - ) - } - - private fun fromSerializable(data: Triple>, List, Timeframe>): Metadata { - val assets = data.second.associateWith { it.deserialize() } - return Metadata(data.first, assets, data.third) - } - - fun clearCache() { - if (cacheFile.exists() && cacheFile.isFile) { - val success = cacheFile.delete() - if (!success) throw RoboquantException("Couldn't delete cache file") - } - } - - - private fun getReader(): DataFileReader { - return DataFileReader(avroFile.toFile(), GenericDatumReader()) - } - - /** - * Build an index of where each time starts. The index helps to achieve faster read access if not starting - * from the beginning. - */ - internal fun build(useCache: Boolean = false): Metadata { - var result = if (useCache) loadFromCache() else null - if (result != null) return result - - var last = Long.MIN_VALUE - val index = mutableListOf>() - var start = Long.MIN_VALUE - var prevPos = Long.MIN_VALUE - - val assetLookup = mutableMapOf() - - getReader().use { - while (it.hasNext()) { - val rec = it.next() - val t = rec[0] as Long - - val assetStr = rec[1].toString() - if (!assetLookup.containsKey(assetStr)) assetLookup[assetStr] = assetStr.deserialize() - if (t > last) { - if (start == Long.MIN_VALUE) start = t - val pos = it.previousSync() - - if (pos != prevPos) { - val time = Instant.ofEpochMilli(t) - index.add(Pair(time, pos)) - prevPos = pos - } - last = t - } - } - } - val timeframe = if (start == Long.MIN_VALUE) - Timeframe.EMPTY - else - Timeframe(Instant.ofEpochMilli(start), Instant.ofEpochMilli(last), true) - - result = Metadata(index, assetLookup, timeframe) - - // We save it for next time only if caching is required - if (useCache) save(result) - return result - } - - private fun ObjectInputStream.isValid(): Boolean { - val avroFileHash = calculateFileHash(avroFile.toFile()) - val hash = readObject() as String - if (hash != avroFileHash) { - AvroFeed.logger.info { "file hash different from found in cache" } - return false - } - - val version = readObject() as String - if (version != VERSION) { - AvroFeed.logger.info { "index file has wrong version" } - return false - } - - return true - } - - private fun loadFromCache(): Metadata? { - if (!cacheFile.exists()) return null - - FileInputStream(cacheFile).use { fileInputStream -> - ObjectInputStream(fileInputStream).use { objectInputStream -> - AvroFeed.logger.info { "loading cache file: $cacheFile" } - - if (!objectInputStream.isValid()) return null - - @Suppress("UNCHECKED_CAST") - val data = objectInputStream.readObject() as Triple>, List, Timeframe> - return fromSerializable(data) - - } - } - - } - - private fun save(result: Metadata) { - AvroFeed.logger.info { "building new cache file: $cacheFile" } - val hash = calculateFileHash(avroFile.toFile()) - - FileOutputStream(cacheFile).use { fileOutputStream -> - ObjectOutputStream(fileOutputStream).use { objOutputStream -> - objOutputStream.writeObject(hash) - objOutputStream.writeObject(VERSION) - objOutputStream.writeObject(toSerializable(result)) - } - } - - } - - companion object { - internal const val VERSION = "1.0" - internal const val CACHE_SUFFIX = ".cache" - } - - /** - * Since files can be large, only read the beginning and ends of the file - * for creating the hash. - */ - private fun readFirstAndLastBytes(file: File): ByteArray { - val bufferSize = 1024 * 1024 // 1 MB - val totalBytesToRead = 2 * bufferSize - val byteArray = ByteArray(totalBytesToRead) - - FileInputStream(file).use { inputStream -> - val bytesRead = inputStream.read(byteArray) - - if (bytesRead < totalBytesToRead) { - // The file is smaller than 2 MB, adjust the array size - return byteArray.copyOfRange(0, bytesRead) - } else { - inputStream.skip(file.length() - bufferSize.toLong()) - inputStream.read(byteArray, bufferSize, bufferSize) - return byteArray - } - } - } - - /** - * Create hash of the file to detect changes - */ - private fun calculateFileHash(file: File): String { - val md = MessageDigest.getInstance("SHA-256") - val bytes = readFirstAndLastBytes(file) - val digest = md.digest(bytes) - return digest.fold("") { str, it -> str + "%02x".format(it) } - } - -} - diff --git a/roboquant-avro/src/main/kotlin/org/roboquant/avro/PriceActionSerializer.kt b/roboquant-avro/src/main/kotlin/org/roboquant/avro/PriceActionSerializer.kt index a2c3da78..97c93747 100644 --- a/roboquant-avro/src/main/kotlin/org/roboquant/avro/PriceActionSerializer.kt +++ b/roboquant-avro/src/main/kotlin/org/roboquant/avro/PriceActionSerializer.kt @@ -26,27 +26,20 @@ import org.roboquant.feeds.* */ internal class PriceActionSerializer { - internal class Serialization(val type: Int, val values: List, val other: String? = null) + internal class Serialization(val type: PriceItemType, val values: List, val other: String? = null) private val timeSpans = mutableMapOf() - private companion object { - private const val PRICEBAR_IDX = 1 - private const val TRADEPRICE_IDX = 2 - private const val PRICEQUOTE_IDX = 3 - private const val ORDERBOOK_IDX = 4 - } - fun serialize(action: PriceItem): Serialization { return when (action) { - is PriceBar -> Serialization(PRICEBAR_IDX, action.ohlcv.toList(), action.timeSpan?.toString()) - is TradePrice -> Serialization(TRADEPRICE_IDX, listOf(action.price, action.volume)) + is PriceBar -> Serialization(PriceItemType.BAR, action.ohlcv.toList(), action.timeSpan?.toString()) + is TradePrice -> Serialization(PriceItemType.TRADE, listOf(action.price, action.volume)) is PriceQuote -> Serialization( - PRICEQUOTE_IDX, + PriceItemType.QUOTE, listOf(action.askPrice, action.askSize, action.bidPrice, action.bidSize) ) - is OrderBook -> Serialization(ORDERBOOK_IDX, orderBookToValues(action)) + is OrderBook -> Serialization(PriceItemType.BOOK, orderBookToValues(action)) else -> throw UnsupportedException("cannot serialize action=$action") } } @@ -61,13 +54,13 @@ internal class PriceActionSerializer { return PriceBar(asset, values, timeSpan) } - fun deserialize(asset: Asset, idx: Int, values: List, other: String?): PriceItem { - return when (idx) { - PRICEBAR_IDX -> getPriceBar(asset, values.toDoubleArray(), other) - TRADEPRICE_IDX -> TradePrice(asset, values[0], values[1]) - PRICEQUOTE_IDX -> PriceQuote(asset, values[0], values[1], values[2], values[3]) - ORDERBOOK_IDX -> getOrderBook(asset, values) - else -> throw UnsupportedException("cannot deserialize asset=$asset type=$idx") + fun deserialize(asset: Asset, type: PriceItemType, values: List, other: String?): PriceItem { + return when (type) { + PriceItemType.BAR -> getPriceBar(asset, values.toDoubleArray(), other) + PriceItemType.TRADE -> TradePrice(asset, values[0], values[1]) + PriceItemType.QUOTE -> PriceQuote(asset, values[0], values[1], values[2], values[3]) + PriceItemType.BOOK -> getOrderBook(asset, values) + else -> throw UnsupportedException("cannot deserialize asset=$asset type=$type") } } diff --git a/roboquant-avro/src/main/kotlin/org/roboquant/avro/avroRecorder.kt b/roboquant-avro/src/main/kotlin/org/roboquant/avro/avroRecorder.kt deleted file mode 100644 index ecded1ba..00000000 --- a/roboquant-avro/src/main/kotlin/org/roboquant/avro/avroRecorder.kt +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright 2020-2024 Neural Layer - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.roboquant.avro - -import kotlinx.coroutines.channels.ClosedReceiveChannelException -import kotlinx.coroutines.launch -import kotlinx.coroutines.runBlocking -import org.apache.avro.Schema -import org.apache.avro.file.CodecFactory -import org.apache.avro.file.DataFileWriter -import org.apache.avro.generic.GenericData -import org.apache.avro.generic.GenericDatumWriter -import org.apache.avro.generic.GenericRecord -import org.apache.avro.io.DatumWriter -import org.roboquant.common.Asset -import org.roboquant.common.AssetFilter -import org.roboquant.common.Timeframe -import org.roboquant.feeds.EventChannel -import org.roboquant.feeds.Feed -import org.roboquant.feeds.PriceItem -import org.roboquant.feeds.util.AssetSerializer.serialize -import java.io.File -import kotlin.io.path.Path - -/** - * Schema used to store different types of [PriceItem] - */ -private const val SCHEMA = """ - { - "namespace": "org.roboquant.avro.schema", - "type": "record", - "name": "PriceItem", - "fields": [ - {"name": "time", "type": "long"}, - {"name": "asset", "type": "string"}, - {"name": "type", "type": "int"}, - {"name": "values", "type": {"type": "array", "items" : "double"}}, - {"name": "other", "type": ["null", "string"], "default": null} - ] - } - """ - -@Suppress("LongParameterList") -internal fun recordAvro( - feed: Feed, - fileName: String, - compression: Boolean = true, - timeframe: Timeframe = Timeframe.INFINITE, - append: Boolean = false, - assetFilter: AssetFilter = AssetFilter.all() -) = runBlocking { - val channel = EventChannel(timeframe = timeframe) - val schema = Schema.Parser().parse(SCHEMA) - val datumWriter: DatumWriter = GenericDatumWriter(schema) - val dataFileWriter = DataFileWriter(datumWriter) - val file = File(fileName) - - val index = MetadataProvider(Path(fileName)) - index.clearCache() - - if (append) { - dataFileWriter.appendTo(file) - } else { - if (compression) dataFileWriter.setCodec(CodecFactory.snappyCodec()) - dataFileWriter.create(schema, file) - } - - val job = launch { - feed.play(channel) - channel.close() - } - - val arraySchema = Schema.createArray(Schema.create(Schema.Type.DOUBLE)) - try { - val cache = mutableMapOf() - val record = GenericData.Record(schema) - val serializer = PriceActionSerializer() - while (true) { - val event = channel.receive() - val now = event.time.toEpochMilli() - for (action in event.items.filterIsInstance() - .filter { assetFilter.filter(it.asset, event.time) }) { - val asset = action.asset - val assetStr = cache.getOrPut(asset) { asset.serialize() } - record.put(0, now) - record.put(1, assetStr) - - val serialization = serializer.serialize(action) - record.put(2, serialization.type) - - val arr = GenericData.Array(serialization.values.size, arraySchema) - arr.addAll(serialization.values) - record.put(3, arr) - - record.put(4, serialization.other) - dataFileWriter.append(record) - } - - } - - } catch (_: ClosedReceiveChannelException) { - // On purpose left empty, expected exception - } finally { - channel.close() - if (job.isActive) job.cancel() - dataFileWriter.sync() - dataFileWriter.close() - } -} diff --git a/roboquant-avro/src/test/kotlin/org/roboquant/TestData.kt b/roboquant-avro/src/test/kotlin/org/roboquant/TestData.kt deleted file mode 100644 index 4eb76a9f..00000000 --- a/roboquant-avro/src/test/kotlin/org/roboquant/TestData.kt +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2020-2024 Neural Layer - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.roboquant - -import org.roboquant.common.Asset -import org.roboquant.common.days -import org.roboquant.common.plus -import org.roboquant.feeds.Event -import org.roboquant.feeds.HistoricFeed -import org.roboquant.feeds.TradePrice -import org.roboquant.feeds.random.RandomWalkFeed -import org.roboquant.feeds.util.HistoricTestFeed -import java.time.Instant - -/** - * Test data used in unit tests - */ -internal object TestData { - - private fun usStock() = Asset("XYZ") - - fun feed(): HistoricFeed { - return HistoricTestFeed(90..110, 110 downTo 80, 80..125, priceBar = true, asset = usStock()) - } - - private fun priceAction(asset: Asset = usStock()) = TradePrice(asset, 10.0) - - private fun time(): Instant = Instant.parse("2020-01-03T12:00:00Z") - - fun event(time: Instant = time()) = Event(time, listOf(priceAction())) - - fun events(n: Int = 100, asset: Asset = usStock()): List { - val start = time() - val result = mutableListOf() - repeat(n) { - val action = TradePrice(asset, it + 100.0) - val event = Event(start + it.days, listOf(action)) - result.add(event) - } - return result - } - - val feed = RandomWalkFeed.lastYears(1, 2) - -} diff --git a/roboquant-avro/src/test/kotlin/org/roboquant/avro/AvroFeedIT.kt b/roboquant-avro/src/test/kotlin/org/roboquant/avro/AvroFeedIT.kt deleted file mode 100644 index 755c29c2..00000000 --- a/roboquant-avro/src/test/kotlin/org/roboquant/avro/AvroFeedIT.kt +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright 2020-2024 Neural Layer - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.roboquant.avro - -import org.junit.jupiter.api.assertDoesNotThrow -import org.roboquant.common.Config -import org.roboquant.common.symbols -import org.roboquant.feeds.PriceBar -import org.roboquant.feeds.PriceQuote -import org.roboquant.feeds.filter -import java.time.Instant -import kotlin.io.path.div -import kotlin.test.* - - -class AvroFeedIT { - - @Test - fun predefinedSP500() { - val feed = AvroFeed.sp500() - assertTrue(feed.assets.size > 490) - assertTrue(feed.timeframe.start >= Instant.parse("2016-01-01T00:00:00Z")) - assertContains(feed.assets.symbols, "AAPL") - assertDoesNotThrow { - var found = false - feed.filter { found = true;false } - assertTrue(found) - } - } - - @Test - fun predefinedQuotes() { - val feed = AvroFeed.sp500Quotes() - assertTrue(feed.assets.size >= 490) - assertContains(feed.assets.symbols, "AAPL") - assertDoesNotThrow { - var found = false - feed.filter { found = true;false } - assertTrue(found) - } - } - - @Test - fun predefinedForex() { - val feed = AvroFeed.forex() - assertEquals(1, feed.assets.size) - assertContains(feed.assets.symbols, "EUR_USD") - assertDoesNotThrow { - var found = false - feed.filter { found = true;false } - assertTrue(found) - } - } - - @Test - fun loadFromStore() { - val fileName = "sp500_pricebar_v6.0.avro" - val file = (Config.home / fileName).toFile() - file.delete() - assertFalse(file.exists()) - - // Force loading of file - AvroFeed.sp500() - val file2 = (Config.home / fileName).toFile() - assertTrue(file2.exists()) - } - - -} diff --git a/roboquant-avro/src/test/kotlin/org/roboquant/avro/AvroFeedTest.kt b/roboquant-avro/src/test/kotlin/org/roboquant/avro/AvroFeedTest.kt index bd9fed01..3b0e7765 100644 --- a/roboquant-avro/src/test/kotlin/org/roboquant/avro/AvroFeedTest.kt +++ b/roboquant-avro/src/test/kotlin/org/roboquant/avro/AvroFeedTest.kt @@ -20,22 +20,17 @@ import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.MethodOrderer.Alphanumeric import org.junit.jupiter.api.TestMethodOrder import org.junit.jupiter.api.assertDoesNotThrow -import org.junit.jupiter.api.assertThrows import org.junit.jupiter.api.io.TempDir -import org.roboquant.TestData import org.roboquant.common.* import org.roboquant.feeds.* import org.roboquant.feeds.random.RandomWalkFeed import org.roboquant.feeds.util.AssetSerializer.deserialize import org.roboquant.feeds.util.AssetSerializer.serialize -import org.roboquant.feeds.util.HistoricTestFeed import java.io.File import java.time.Instant import java.util.* -import kotlin.io.path.Path import kotlin.test.Test import kotlin.test.assertEquals -import kotlin.test.assertFalse import kotlin.test.assertTrue @TestMethodOrder(Alphanumeric::class) @@ -64,68 +59,22 @@ internal class AvroFeedTest { @TempDir lateinit var folder: File - private lateinit var fileName: String - private var size: Int = 0 - private const val NR_ASSETS = 2 - var assets = mutableListOf() - } - - @Test - fun avroStep1() { - fileName = File(folder, "test.avro").path - val feed = TestData.feed - assets.addAll(feed.assets) - size = feed.toList().size - AvroFeed.record(feed, fileName) - assertTrue(File(fileName).isFile) - } + private val fileName: String + get() = File(folder, "test2.avro").path.toString() - @Test - fun avroStep2() { - val feed2 = AvroFeed(Path(fileName)) - runBlocking { - var past = Instant.MIN - var cnt = 0 - for (event in play(feed2)) { - assertTrue(event.time > past) - assertEquals(NR_ASSETS, event.items.size) - past = event.time - cnt++ - } - assertEquals(size, cnt) - } } - @Test - fun cache() { - val fileName = File(folder, "test2.avro").path - val feed = TestData.feed - assets.addAll(feed.assets) - size = feed.toList().size - AvroFeed.record(feed, fileName) - assertTrue(File(fileName).isFile) - AvroFeed(Path(fileName), useCache = true) - val file = File(fileName + MetadataProvider.CACHE_SUFFIX) - assertTrue(file.isFile) - val index = MetadataProvider(Path(fileName)) - index.clearCache() - assertFalse(file.exists()) - - } @Test fun feedPlayback() { val feed3 = AvroFeed(fileName) - assertEquals(NR_ASSETS, feed3.assets.size) - assertEquals(assets.toSet(), feed3.assets.toSet()) assertTrue(feed3.timeframe.inclusive) runBlocking { var cnt = 0 for (event in play(feed3)) cnt++ - assertEquals(size, cnt) } } @@ -147,34 +96,17 @@ internal class AvroFeedTest { feed.events.add(Event(now + 3.millis, listOf(p3))) feed.events.add(Event(now + 4.millis, listOf(p4))) + val feed2 = AvroFeed(fileName) assertDoesNotThrow { - AvroFeed.record(feed, fileName) + feed2.record(feed) } - val feed2 = AvroFeed(fileName) val actions = feed2.filter().map { it.second } assertEquals(4, actions.size) } - @Test - fun unsupportedPriceAction() { - class MyPrice(override val asset: Asset, override val volume: Double) : PriceItem { - override fun getPrice(type: String): Double { - return 10.0 - } - } - - val asset = Asset("DUMMY") - val p1 = MyPrice(asset, 100.0) - val feed = MyFeed(sortedSetOf(asset)) - feed.events.add(Event(Instant.now(), listOf(p1))) - - assertThrows { - AvroFeed.record(feed, fileName) - } - } @Test fun append() { @@ -182,33 +114,15 @@ internal class AvroFeedTest { val past = Timeframe(now - 2.years, now - 1.years) val feed = RandomWalkFeed(past, 1.days) val fileName = File(folder, "test2.avro").path - - AvroFeed.record(feed, fileName, compression = true) - var avroFeed = AvroFeed(fileName) - assertEquals(feed.assets, avroFeed.assets) + val feed2 = AvroFeed(fileName) + feed2.record(feed, compress = true) val past2 = Timeframe(now - 1.years, now) - val feed2 = RandomWalkFeed(past2, 1.days) - AvroFeed.record(feed2, fileName, append = true) - avroFeed = AvroFeed(fileName) - assertEquals(feed.assets + feed2.assets, avroFeed.assets) + val feed3 = RandomWalkFeed(past2, 1.days) + feed2.record(feed3, append = true) } - @Test - fun timeSpan() { - val feed = HistoricTestFeed(priceBar = true) - val pb = feed.toList().first().items.first() - assertTrue(pb is PriceBar) - assertEquals(1.days, pb.timeSpan) - - val fileName = File(folder, "test_timespan.avro").path - AvroFeed.record(feed, fileName, compression = true) - - val avroFeed = AvroFeed(fileName) - val pb2 = avroFeed.toList().first().items.first() - assertTrue(pb2 is PriceBar) - assertEquals(1.days, pb2.timeSpan) - } + @Test fun assetSerialization() { @@ -218,10 +132,9 @@ internal class AvroFeedTest { val asset2 = str.deserialize() assertEquals(asset1, asset2) - val asset3 = - Asset("XYZ", AssetType.BOND, currencyCode = "EUR", exchangeCode = "AEB", multiplier = 2.0, id = "123") + val asset3 = Asset("XYZ", AssetType.BOND, currencyCode = "EUR", exchangeCode = "AEB", id = "123") val str3 = asset3.serialize() - assertEquals("XYZ\u001FBOND\u001FEUR\u001FAEB\u001F2.0\u001F123", str3) + assertEquals("XYZ\u001FBOND\u001FEUR\u001FAEB\u001F123", str3) val asset4 = str3.deserialize() assertEquals(asset3, asset4) diff --git a/roboquant-avro/src/test/kotlin/org/roboquant/samples/AvroSamples.kt b/roboquant-avro/src/test/kotlin/org/roboquant/samples/AvroSamples.kt index ec76d013..5ec1ce53 100644 --- a/roboquant-avro/src/test/kotlin/org/roboquant/samples/AvroSamples.kt +++ b/roboquant-avro/src/test/kotlin/org/roboquant/samples/AvroSamples.kt @@ -20,16 +20,15 @@ import kotlinx.coroutines.channels.ClosedReceiveChannelException import kotlinx.coroutines.runBlocking import org.roboquant.Roboquant import org.roboquant.avro.AvroFeed -import org.roboquant.avro.AvroFeed2 import org.roboquant.brokers.Account import org.roboquant.brokers.FixedExchangeRates import org.roboquant.brokers.sim.MarginAccount -import org.roboquant.brokers.sim.NoCostPricingEngine import org.roboquant.brokers.sim.SimBroker import org.roboquant.brokers.summary import org.roboquant.common.* -import org.roboquant.feeds.* -import org.roboquant.feeds.csv.CSVConfig +import org.roboquant.feeds.Event +import org.roboquant.feeds.EventChannel +import org.roboquant.feeds.PriceItemType import org.roboquant.feeds.csv.CSVFeed import org.roboquant.feeds.csv.PriceBarParser import org.roboquant.feeds.csv.TimeParser @@ -43,16 +42,13 @@ import org.roboquant.policies.resolve import org.roboquant.strategies.CombinedStrategy import org.roboquant.strategies.EMAStrategy import org.roboquant.strategies.Signal -import java.io.File import java.time.Instant import java.time.LocalDateTime import java.time.format.DateTimeFormatter -import kotlin.io.path.Path import kotlin.io.path.div import kotlin.system.measureTimeMillis import kotlin.test.Ignore import kotlin.test.Test -import kotlin.test.assertEquals import kotlin.test.assertTrue internal class AvroSamples { @@ -60,11 +56,11 @@ internal class AvroSamples { @Test @Ignore internal fun quotes() = runBlocking { - val f = RandomWalkFeed.lastDays(20) - val tf1 = f.timeline.timeframe - val feed = AvroFeed2("/tmp/test.avro") - feed.record(f) - assertEquals(tf1, feed.timeframe) + val tf = Timeframe.parse("2021-01-01", "2023-01-01") + val f = RandomWalkFeed(tf, 1.seconds, nAssets = 1, priceType = PriceItemType.QUOTE) + val feed = AvroFeed("/tmp/test.avro") + feed.record(f, compress = false) + assertTrue(feed.exists()) val channel = EventChannel() feed.playBackground(channel) @@ -163,54 +159,9 @@ internal class AvroSamples { roboquant.run(feed, timeframe = Timeframe.past(5.years)) } - @Test - @Ignore - internal fun simple() { - val strategy = EMAStrategy() - val feed = AvroFeed.sp500() - val roboquant = Roboquant(strategy) - val account = roboquant.run(feed) - println(account.fullSummary()) - } - @Test - @Ignore - internal fun aggregator() { - val forex = AvroFeed.forex() - val feed = AggregatorFeed(forex, 15.minutes) - feed.apply { _, time -> - println(time) - } - } - @Test - @Ignore - internal fun forexRun() { - val feed = AvroFeed.forex() - Currency.increaseDigits(3) - val rq = Roboquant(EMAStrategy(), broker = SimBroker(pricingEngine = NoCostPricingEngine())) - val account = rq.run(feed, timeframe = Timeframe.parse("2022-01-03", "2022-02-10")) - - for (trade in account.trades) { - val tf = Timeframe(trade.time, trade.time, true) - val pricebar = feed.filter(timeframe = tf).firstOrNull { it.second.asset == trade.asset } - if (pricebar == null) { - println(trade) - println(feed.filter(timeframe = tf)) - throw RoboquantException("couldn't find trade action") - } else { - assertEquals(pricebar.second.getPrice(), trade.price) - } - } - } - @Test - @Ignore - internal fun profileTest() { - val feed = AvroFeed.sp500() - val rq = Roboquant(EMAStrategy()) - rq.run(feed) - } @Test @Ignore @@ -273,63 +224,8 @@ internal class AvroSamples { println(account.trades.summary()) } - @Test - @Ignore - internal fun feedRecorder() { - val tf = Timeframe.past(1.years) - val symbol = "BTCBUSD" - val template = Asset(symbol, AssetType.CRYPTO, currency = Currency.getInstance("BUSD")) - val feed = RandomWalkFeed(tf, 1.seconds, template = template, nAssets = 1, generateBars = false) - val fileName = "/tmp/${symbol}-1sec.avro" - val t = measureTimeMillis { - AvroFeed.record(feed, fileName) - } - val f = File(fileName) - println(t) - println(f.length() / 1_000_000) - - val t2 = measureTimeMillis { - AvroFeed(fileName) - } - println(t2) - } - - @Test - @Ignore - internal fun generateDemoFeed() { - val pathStr = Config.getProperty("datadir", "/tmp/us") - val timeframe = Timeframe.fromYears(2014, 2024) - val symbols = Universe.sp500.getAssets(timeframe.end).map { it.symbol }.toTypedArray() - assertTrue(symbols.size > 490) - - val config = CSVConfig.stooq() - val path = Path(pathStr) - val path1 = path / "nasdaq stocks" - val path2 = path / "nyse stocks" - - val feed = CSVFeed(path1.toString(), config) - val tmp = CSVFeed(path2.toString(), config) - feed.merge(tmp) - - val sp500File = "/tmp/sp500_pricebar_v6.1.avro" - - AvroFeed.record( - feed, - sp500File, - true, - timeframe, - assetFilter = AssetFilter.includeSymbols(*symbols) - ) - - // Some basic sanity checks that recording went ok - val avroFeed = AvroFeed(sp500File) - assertTrue(avroFeed.assets.size > 490) - assertTrue(avroFeed.assets.symbols.contains("AAPL")) - assertTrue(avroFeed.timeframe > 4.years) - - } } diff --git a/roboquant-charts/src/test/kotlin/org/roboquant/charts/ChartTest.kt b/roboquant-charts/src/test/kotlin/org/roboquant/charts/ChartTest.kt index e7a3b2d0..cf9d262b 100644 --- a/roboquant-charts/src/test/kotlin/org/roboquant/charts/ChartTest.kt +++ b/roboquant-charts/src/test/kotlin/org/roboquant/charts/ChartTest.kt @@ -45,7 +45,7 @@ internal class ChartTest { @Test fun test() { - val f = RandomWalkFeed.lastYears(1, 1, generateBars = true) + val f = RandomWalkFeed.lastYears(1, 1) val asset = f.assets.first() val chart = PriceBarChart(f, asset) val html = chart.renderJson() diff --git a/roboquant-ibkr/src/main/kotlin/org/roboquant/ibkr/IBKR.kt b/roboquant-ibkr/src/main/kotlin/org/roboquant/ibkr/IBKR.kt index 79291c49..abb3da32 100644 --- a/roboquant-ibkr/src/main/kotlin/org/roboquant/ibkr/IBKR.kt +++ b/roboquant-ibkr/src/main/kotlin/org/roboquant/ibkr/IBKR.kt @@ -125,7 +125,7 @@ object IBKR { contract.symbol(symbol) contract.currency(currency.currencyCode) - if (multiplier != 1.0) contract.multiplier(multiplier.toString()) + // if (multiplier != 1.0) contract.multiplier(multiplier.toString()) when (type) { AssetType.STOCK -> contract.secType(Types.SecType.STK) diff --git a/roboquant-jupyter/src/main/kotlin/org/roboquant/jupyter/Welcome.kt b/roboquant-jupyter/src/main/kotlin/org/roboquant/jupyter/Welcome.kt index cd39265a..c87521fe 100644 --- a/roboquant-jupyter/src/main/kotlin/org/roboquant/jupyter/Welcome.kt +++ b/roboquant-jupyter/src/main/kotlin/org/roboquant/jupyter/Welcome.kt @@ -16,12 +16,7 @@ package org.roboquant.jupyter -import org.roboquant.Roboquant -import org.roboquant.avro.AvroFeed -import org.roboquant.charts.Chart -import org.roboquant.charts.PriceBarChart import org.roboquant.common.Config -import org.roboquant.strategies.EMAStrategy /** * Provides current environment settings in HTML format suitable for displaying in a Jupyter Notebook. @@ -63,53 +58,5 @@ class Welcome { """.trimIndent() } - /** - * Run a small demo back test and display the resulting equity curve - */ - fun demo1() { - val strategy = EMAStrategy() - val roboquant = Roboquant(strategy) - val feed = AvroFeed.sp500() - println( - """ - ┌───────────────┐ - │ INPUT │ - └───────────────┘ - val strategy = EMAStrategy() - val roboquant = Roboquant(strategy) - - val feed = AvroFeed.sp500() - roboquant.run(feed) - ┌───────────────┐ - │ Output │ - └───────────────┘ - """.trimIndent() - ) - - roboquant.run(feed) - } - - - /** - * View feed data demo - */ - fun demo2(): Chart { - println( - """ - ┌───────────────┐ - │ INPUT │ - └───────────────┘ - val feed = AvroFeed.sp500() - PriceBarChart(feed, "AAPL") - - ┌───────────────┐ - │ Output │ - └───────────────┘ - """.trimIndent() - ) - - val feed = AvroFeed.sp500() - return PriceBarChart(feed, "AAPL") - } } diff --git a/roboquant-jupyter/src/test/kotlin/org/roboquant/jupyter/WelcomeTestIT.kt b/roboquant-jupyter/src/test/kotlin/org/roboquant/jupyter/WelcomeTestIT.kt index ac96a7cf..e9ec8b0e 100644 --- a/roboquant-jupyter/src/test/kotlin/org/roboquant/jupyter/WelcomeTestIT.kt +++ b/roboquant-jupyter/src/test/kotlin/org/roboquant/jupyter/WelcomeTestIT.kt @@ -16,8 +16,6 @@ package org.roboquant.jupyter -import org.junit.jupiter.api.assertDoesNotThrow -import org.roboquant.charts.PriceBarChart import kotlin.test.Test import kotlin.test.assertTrue @@ -31,20 +29,4 @@ internal class WelcomeTestIT { assertTrue { w.asHTMLPage().contains(snippet) } } - - @Test - fun testDemo1() { - assertDoesNotThrow { - Welcome().demo1() - } - } - - - @Test - fun testDemo2() { - val chart2 = Welcome().demo2() - assertTrue(chart2 is PriceBarChart) - } - - } diff --git a/roboquant-ml/pom.xml b/roboquant-ml/pom.xml deleted file mode 100644 index 2a06859b..00000000 --- a/roboquant-ml/pom.xml +++ /dev/null @@ -1,89 +0,0 @@ - - - - - 4.0.0 - - roboquant-parent - org.roboquant - 3.0.0-SNAPSHOT - - - roboquant-ml - jar - roboquant ml - Machine Learningsupport for the roboquant algorithmic trading platform - - - 3.0.2 - - - - - - org.jetbrains.kotlin - kotlin-maven-plugin - - - - - - - org.roboquant - roboquant - - - org.roboquant - roboquant-ta - ${project.version} - - - org.bytedeco - openblas - 0.3.21-1.5.8 - - - org.bytedeco - openblas-platform - 0.3.21-1.5.8 - - - com.github.haifengl - smile-core - ${smile.version} - - - com.github.haifengl - smile-kotlin - ${smile.version} - - - io.deephaven - SuanShu - 0.1.0 - - - org.roboquant - roboquant-avro - ${project.version} - test - - - - diff --git a/roboquant-ml/src/main/kotlin/org/roboquant/ml/DataFrameFeatureSet.kt b/roboquant-ml/src/main/kotlin/org/roboquant/ml/DataFrameFeatureSet.kt deleted file mode 100644 index 508195a2..00000000 --- a/roboquant-ml/src/main/kotlin/org/roboquant/ml/DataFrameFeatureSet.kt +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright 2020-2024 Neural Layer - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.roboquant.ml - -import org.roboquant.feeds.Event -import smile.data.DataFrame -import smile.data.vector.DoubleVector - -/** - * FeatureSet contains one or more [features][Feature] and can return it as a dataframe - */ -class DataFrameFeatureSet(private val historySize: Int = 1_000_000, private val warmup: Int = 0) { - - private class Entry( - val feature: Feature, - val matrix: Matrix, - val offset: Int = 0, - // var mean: Double = Double.NaN - ) { - fun inputData(lastRow: Int): DoubleArray { - val len = lastRow*feature.size - val result = DoubleArray(len) - System.arraycopy(matrix.data, 0, result, 0, len) - // mean = result.fillNaNMean() - return result - } - - fun toDoubleVector(lastRow: Int): DoubleVector { - return DoubleVector.of(feature.name, inputData(lastRow)) - } - - } - - private val entries = mutableListOf() - - private var samples = 0 - private var warmupCountdown = warmup - - /** - * The names of all the features in this feature set - */ - val names - get() = entries.map { it.feature.name } - - private val maxOffset - get() = entries.maxOf { it.offset } - - /** - * Add one or more [features] and optionally provide an [offset] - */ - fun add(vararg features: Feature, offset: Int = 0) { - for (f in features) { - require(f.name !in names) { "duplicate feature name ${f.name}" } - entries.add(Entry(f, Matrix(historySize, f.size), offset)) - } - } - - - /** - * Returns the training data as a [DataFrame] - */ - fun getTrainingData(): DataFrame { - val size = samples - maxOffset - val i = entries.map {it.toDoubleVector(size) } - @Suppress("SpreadOperator") - return DataFrame.of(*i.toTypedArray()) - } - - - private fun Entry.getRow(row: Int): DoubleVector { - val last = samples - offset - 1 - val doubleArr = if (row > last) doubleArrayOf(Double.NaN) else matrix[row] - return DoubleVector.of(feature.name, doubleArr) - } - - /** - * Returns the prediction data as a [DataFrame] - */ - fun getPredictData(row: Int = samples - 1): DataFrame { - assert(row >= 0) - val i = entries.map { it.getRow(row) } - @Suppress("SpreadOperator") - return DataFrame.of(*i.toTypedArray()) - } - - /** - * Update all the features in this set the provided [event] and store the results internally - */ - fun update(event: Event) { - if (warmupCountdown > 0) { - for (entry in entries) entry.feature.calculate(event) - warmupCountdown-- - return - } - - for (entry in entries) { - val value = entry.feature.calculate(event) - val idx = samples - entry.offset - if (idx >= 0) { - entry.matrix[idx] = value - } - } - samples++ - } - - /** - * Reset internal state and all features - */ - fun reset() { - for (f in entries) f.feature.reset() - samples = 0 - warmupCountdown = warmup - } - - -} diff --git a/roboquant-ml/src/main/kotlin/org/roboquant/ml/DivFeature.kt b/roboquant-ml/src/main/kotlin/org/roboquant/ml/DivFeature.kt deleted file mode 100644 index 6947331f..00000000 --- a/roboquant-ml/src/main/kotlin/org/roboquant/ml/DivFeature.kt +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2020-2024 Neural Layer - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.roboquant.ml - -import org.roboquant.feeds.Event - -/** - * Create a new feature based on the division of two other features. - */ -class DivFeature(private val numerator: SingleValueFeature, private val denominator: SingleValueFeature) : - SingleValueFeature() { - - /** - * @see Feature.calculate - */ - override fun calculateValue(event: Event): Double { - return numerator.calculateValue(event) / denominator.calculateValue(event) - } - - /** - * Returns the name of this feature - */ - override val name: String - get() = numerator.name + "-DIV-" + denominator.name - - /** - * reset the underlying [numerator] and [denominator] features - */ - override fun reset() { - numerator.reset() - denominator.reset() - } -} - -/** - * Divide two features and return this new feature. - * - * ``` - * val feature = PriceFeature(asset, "OPEN") / PriceFeature(asset, "CLOSE") - * ``` - * @see DivFeature - */ -operator fun SingleValueFeature.div(denominator: SingleValueFeature): Feature = DivFeature(this, denominator) diff --git a/roboquant-ml/src/main/kotlin/org/roboquant/ml/Feature.kt b/roboquant-ml/src/main/kotlin/org/roboquant/ml/Feature.kt deleted file mode 100644 index 05ac19bf..00000000 --- a/roboquant-ml/src/main/kotlin/org/roboquant/ml/Feature.kt +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2020-2024 Neural Layer - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.roboquant.ml - -import org.roboquant.feeds.Event - - -/** - * A feature generates data that is derived from a series of events - */ -interface Feature { - - /** - * Update the feature with a new event and return the latest value - */ - fun calculate(event: Event): DoubleArray - - /** - * The name of this feature - */ - val name: String - - /** - * The size of the returned DoubleArray. This should be the same size at every event. - */ - val size: Int - - /** - * Reset any state in the feature - */ - fun reset() {} -} - - -/** - * A feature generates a single value that is derived from a series of events - */ -abstract class SingleValueFeature: Feature { - - override val size: Int = 1 - - /** - * Update the feature with a new event and return the latest value - */ - override fun calculate(event: Event): DoubleArray { - return doubleArrayOf(calculateValue(event)) - } - - abstract fun calculateValue(event: Event): Double - -} - diff --git a/roboquant-ml/src/main/kotlin/org/roboquant/ml/HistoricPriceFeature.kt b/roboquant-ml/src/main/kotlin/org/roboquant/ml/HistoricPriceFeature.kt deleted file mode 100644 index 0c2b022a..00000000 --- a/roboquant-ml/src/main/kotlin/org/roboquant/ml/HistoricPriceFeature.kt +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2020-2024 Neural Layer - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.roboquant.ml - -import org.roboquant.common.Asset -import org.roboquant.feeds.Event - -/** - * Extract a historic price from the event for the provided [asset] - * - * @param asset the asset to use - * @param past how many events in the past should be used - * @param type the type of price to use, default is "DEFAULT" - * @param name the name of the feature - */ -class HistoricPriceFeature( - private val asset: Asset, - private val past: Int = 1, - private val type: String = "DEFAULT", - override val name: String = "${asset.symbol}-HISTORIC-PRICE-$past-$type" -) : SingleValueFeature() { - - private val hist = mutableListOf() - - /** - * @see Feature.calculate - */ - override fun calculateValue(event: Event): Double { - val action = event.prices[asset] - hist.add(action?.getPrice(type) ?: Double.NaN) - return if (hist.size > past) hist.removeFirst() else Double.NaN - } - - override fun reset() { - hist.clear() - } - -} diff --git a/roboquant-ml/src/main/kotlin/org/roboquant/ml/Matrix.kt b/roboquant-ml/src/main/kotlin/org/roboquant/ml/Matrix.kt deleted file mode 100644 index aeaaf93d..00000000 --- a/roboquant-ml/src/main/kotlin/org/roboquant/ml/Matrix.kt +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2020-2024 Neural Layer - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.roboquant.ml - -class Matrix(private val rows: Int, private val columns: Int) { - - internal val data = DoubleArray(rows * columns) {Double.NaN} - - operator fun set(row: Int, value: DoubleArray) { - System.arraycopy(value, 0, data, row * columns, columns) - } - - operator fun get(row: Int): DoubleArray { - assert(row < rows) - val result = DoubleArray(columns) {Double.NaN} - System.arraycopy(data, row*columns, result, 0, columns) - return result - } - - operator fun get(start: Int, end: Int): DoubleArray { - assert(end < rows) - val len = end - start - val result = DoubleArray(columns * len) {Double.NaN} - System.arraycopy(data, start*columns, result, 0, len*columns) - return result - } - -} diff --git a/roboquant-ml/src/main/kotlin/org/roboquant/ml/PriceFeature.kt b/roboquant-ml/src/main/kotlin/org/roboquant/ml/PriceFeature.kt deleted file mode 100644 index a93d4738..00000000 --- a/roboquant-ml/src/main/kotlin/org/roboquant/ml/PriceFeature.kt +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2020-2024 Neural Layer - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.roboquant.ml - -import org.roboquant.common.Asset -import org.roboquant.feeds.Event - -/** - * Extract the price from the event for the provided [asset] - * - * @param asset the asset to use - * @param type the type of price to use - * @param name the name of the feature - */ -class PriceFeature( - private val asset: Asset, - private val type: String = "DEFAULT", - override val name: String = "${asset.symbol}-PRICE-$type" -) : SingleValueFeature() { - - /** - * @see SingleValueFeature.calculateValue - */ - override fun calculateValue(event: Event): Double { - val action = event.prices[asset] - return action?.getPrice(type) ?: Double.NaN - } - - -} diff --git a/roboquant-ml/src/main/kotlin/org/roboquant/ml/RegressionStrategy.kt b/roboquant-ml/src/main/kotlin/org/roboquant/ml/RegressionStrategy.kt deleted file mode 100644 index 4ce29172..00000000 --- a/roboquant-ml/src/main/kotlin/org/roboquant/ml/RegressionStrategy.kt +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright 2020-2024 Neural Layer - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.roboquant.ml - -import org.roboquant.common.Asset -import org.roboquant.common.Logging -import org.roboquant.common.addNotNull -import org.roboquant.common.percent -import org.roboquant.feeds.Event -import org.roboquant.strategies.Signal -import org.roboquant.strategies.Strategy -import smile.data.DataFrame -import smile.regression.DataFrameRegression - -/** - * Strategy based on a Smile Regression - */ -open class RegressionStrategy( - private val featureSet: DataFrameFeatureSet, - private val asset: Asset, - private val percentage: Double = 1.percent, - val block: (DataFrame) -> DataFrameRegression -) : Strategy { - - private val logger = Logging.getLogger(this::class) - private var trained = false - - private lateinit var model: DataFrameRegression - - private fun train() { - val df = featureSet.getTrainingData() - model = block(df) - } - - private fun predict(): Double { - val df = featureSet.getPredictData() - val result = model.predict(df).last() - return result - } - - override fun generate(event: Event): List { - featureSet.update(event) - val results = mutableListOf() - if (trained) { - val pred = predict() - val signal = getSignal(asset, pred, event) - results.addNotNull(signal) - } - return results - } - - /** - * Allow custom logic for generating more advanced signals - */ - open fun getSignal(asset: Asset, prediction: Double, event: Event) : Signal? { - return when { - prediction > percentage -> Signal(asset, prediction) - prediction < - percentage -> Signal(asset, prediction) - else -> null - } - } - - - @Suppress("unused", "unused") - fun eanbleTrain() { - if (! trained) { - logger.trace { "start training" } - train() - trained = true - } - } - - override fun reset() { - featureSet.reset() - trained = false - } - -} diff --git a/roboquant-ml/src/main/kotlin/org/roboquant/ml/ReturnsFeature.kt b/roboquant-ml/src/main/kotlin/org/roboquant/ml/ReturnsFeature.kt deleted file mode 100644 index 7e4b7af2..00000000 --- a/roboquant-ml/src/main/kotlin/org/roboquant/ml/ReturnsFeature.kt +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2020-2024 Neural Layer - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.roboquant.ml - -import org.roboquant.common.div -import org.roboquant.common.minus -import org.roboquant.feeds.Event - -/** - * - */ -class ReturnsFeature( - private val f: Feature, - private val n: Int = 1, - private val missingValue: Double = Double.NaN, - override val name: String = f.name + "-RETURNS" -) : Feature { - - private val history = mutableListOf() - private val missing = DoubleArray(f.size) { missingValue } - override val size = f.size - - /** - * @see Feature.calculate - */ - override fun calculate(event: Event): DoubleArray { - val v = f.calculate(event) - history.add(v) - return if (history.size > n) { - val first = history.removeFirst() - history.last() / first - 1.0 - } else { - missing - } - } - - - override fun reset() { - history.clear() - f.reset() - } - -} - -/** - * Wrap the feature and calculate the returns - */ -fun SingleValueFeature.returns(n: Int = 1) = ReturnsFeature(this, n) - diff --git a/roboquant-ml/src/main/kotlin/org/roboquant/ml/TaLibFeature.kt b/roboquant-ml/src/main/kotlin/org/roboquant/ml/TaLibFeature.kt deleted file mode 100644 index 925826d9..00000000 --- a/roboquant-ml/src/main/kotlin/org/roboquant/ml/TaLibFeature.kt +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright 2020-2024 Neural Layer - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.roboquant.ml - -import org.roboquant.common.Asset -import org.roboquant.feeds.Event -import org.roboquant.feeds.PriceBar -import org.roboquant.ta.InsufficientData -import org.roboquant.ta.PriceBarSeries -import org.roboquant.ta.TaLib - -/** - * Use any TaLib indicators to create features - */ -class TaLibFeature( - override val name: String, - private val asset: Asset, - private val missing: Double = Double.NaN, - private val block: TaLib.(prices: PriceBarSeries) -> Double -) : SingleValueFeature() { - - private val taLib = TaLib() - private val history = PriceBarSeries(1) - - /** - * Common - */ - companion object { - - /** - * RSI feature - */ - fun rsi(asset: Asset, timePeriod: Int = 14): TaLibFeature { - return TaLibFeature("${asset.symbol}-RSI-$timePeriod", asset) { - rsi(it, timePeriod) - } - } - - /** - * OBV feature - */ - fun obv(asset: Asset) : TaLibFeature { - return TaLibFeature("${asset.symbol}-OBV", asset) { - obv(it.close, it.volume) - } - } - - /** - * EMA feature - */ - fun ema(asset: Asset, fast: Int = 5, slow: Int = 13) : TaLibFeature { - return TaLibFeature("${asset.symbol}-EMA-$fast-$slow", asset) { - val f = ema(it, fast) - val s = ema(it, slow) - when { - f > s -> 1.0 - s < f -> -1.0 - else -> 0.0 - } - } - } - - } - - /** - * @see Feature.calculate - */ - override fun calculateValue(event: Event): Double { - val action = event.prices[asset] - if (action != null && action is PriceBar && history.add(action, event.time)) { - try { - return taLib.block(history) - } catch (e: InsufficientData) { - history.increaseCapacity(e.minSize) - } - } - return missing - } - - /** - * @see Feature.reset - */ - override fun reset() { - history.clear() - } - - - -} diff --git a/roboquant-ml/src/main/kotlin/org/roboquant/ml/TestFeature.kt b/roboquant-ml/src/main/kotlin/org/roboquant/ml/TestFeature.kt deleted file mode 100644 index 3a6c1e90..00000000 --- a/roboquant-ml/src/main/kotlin/org/roboquant/ml/TestFeature.kt +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2020-2024 Neural Layer - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.roboquant.ml - -import org.roboquant.feeds.Event - -/** - * test feature - */ -@Suppress("unused") -class TestFeature( - private val value: DoubleArray, - override val name: String = "TEST-FEATURE" -) : Feature { - - override val size: Int = value.size - - /** - * @see Feature.calculate - */ - override fun calculate(event: Event): DoubleArray { - return value - } - - -} diff --git a/roboquant-ml/src/main/kotlin/org/roboquant/ml/VolumeFeature.kt b/roboquant-ml/src/main/kotlin/org/roboquant/ml/VolumeFeature.kt deleted file mode 100644 index 8b4cd7fb..00000000 --- a/roboquant-ml/src/main/kotlin/org/roboquant/ml/VolumeFeature.kt +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2020-2024 Neural Layer - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.roboquant.ml - -import org.roboquant.common.Asset -import org.roboquant.feeds.Event - -/** - * Extract the volume from the price-action for the [asset] - */ -class VolumeFeature( - private val asset: Asset, - override val name: String = "${asset.symbol}-VOLUME" -) : SingleValueFeature() { - - /** - * @see Feature.calculate - */ - override fun calculateValue(event: Event): Double { - val action = event.prices[asset] - return action?.volume ?: Double.NaN - } - - -} diff --git a/roboquant-ml/src/main/kotlin/org/roboquant/ml/extensions.kt b/roboquant-ml/src/main/kotlin/org/roboquant/ml/extensions.kt deleted file mode 100644 index 708ee426..00000000 --- a/roboquant-ml/src/main/kotlin/org/roboquant/ml/extensions.kt +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright 2020-2024 Neural Layer - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -@file:Suppress("unused") - -package org.roboquant.ml - -import org.roboquant.common.Config - - -/** - * Drop first [n] elements from the array and return the result - */ -fun DoubleArray.drop(n: Int = 1): DoubleArray { - if (n == 0) return this - val newSize = size - n - val data = DoubleArray(newSize) - System.arraycopy(this, n, data, 0, newSize) - return data -} - -/** - * Concatenate a number of arrays - */ -fun concatenate(vararg arrays: DoubleArray): DoubleArray = concatenate(arrays.toList()) - -/** - * Concatenate a number of arrays and return the result. The arrays can be of different size - */ -fun concatenate(arrays: Collection): DoubleArray { - if (arrays.isEmpty()) return DoubleArray(0) - - val size = arrays.sumOf { it.size } - val data = DoubleArray(size) - var offset = 0 - arrays.forEach { arr -> - val s = arr.size - System.arraycopy(arr, 0, data, offset, s) - offset += s - } - - return data -} - -/** - * Sample [n] elements of [size] from the array and return the result. The result is return as a list of rows - */ -fun DoubleArray.sample(size: Int, n: Int = 1) = buildList { - val r = Config.random - val max = this.size - size - repeat(n) { - val offset = r.nextInt(max) - val arr = DoubleArray(size) - System.arraycopy(this@sample, offset, arr, 0, size) - add(arr) - } -} - - -class Sampler(private val data: DoubleArray, private val size: Int) { - - private val r = Config.random - private val max = this.size - size - - fun sample(): DoubleArray { - val arr = DoubleArray(size) - val offset = r.nextInt(max) - System.arraycopy(this, offset, arr, 0, size) - return arr - } - -} - -/** - * Replace all NaN values with the mean. - * There should be at least one finite number in the array - */ -internal fun DoubleArray.fillNaNMean(size: Int = this.size): Double { - var n = 0 - var sum = 0.0 - for (idx in 0.. { - val result = mutableListOf() - val max = this.size - size - repeat(size) { - result.add(DoubleArray(n)) - } - val r = Config.random - repeat(n) { - val offset = r.nextInt(max) - for (i in 0..() - feed.applyEvents { event -> - val prices = assets.map { event.getPrice(it) ?: Double.NaN } - if (prices.all { it.isFinite() }) result.add(prices.toDoubleArray()) - } - return SimpleMultiVariateTimeSeries(*result.toTypedArray()) - -} - -/** - * The HIGH-LOW values for a single asset should be conintegrated - */ -private fun getTimeSeriesHighLow(feed: AssetFeed): SimpleMultiVariateTimeSeries { - val apple = feed.assets.getBySymbol("AAPL") - val result = mutableListOf() - feed.applyEvents { event -> - val price = event.prices[apple] - if (price != null) result.add(doubleArrayOf(price.getPrice("HIGH"), price.getPrice("LOW"))) - - } - return SimpleMultiVariateTimeSeries(*result.toTypedArray()) - -} - -fun runJohansenTest(ts: SimpleMultiVariateTimeSeries) { - // Lets do the cointegration - val coint = CointegrationMLE(ts, true, 2) - - // Lets test if they are actually cointegratedd - val test = JohansenTest( - JohansenAsymptoticDistribution.Test.EIGEN, - JohansenAsymptoticDistribution.TrendType.RESTRICTED_CONSTANT, - coint.rank() - ) - println("JohansenTest statistics: ${test.getStats(coint)}") - println("JohansenTest r@0.05: ${test.r(coint, 5.percent)}") -} - - - -fun testStrat(feed: AssetFeed) { - - class PairTradingStrategy(val period: Int, val priceType: String = "DEFAULT") : Strategy { - - - /** - * Contains the history of all assets - */ - private val history = mutableMapOf() - - - fun johansenTest(data: Array): Int { - val ts = SimpleMultiVariateTimeSeries(*data) - val coint = CointegrationMLE(ts, true, 2) - - // Lets test if they are actually cointegratedd - val test = JohansenTest( - JohansenAsymptoticDistribution.Test.EIGEN, - JohansenAsymptoticDistribution.TrendType.RESTRICTED_CONSTANT, - coint.rank() - ) - val result = test.r(coint, 5.percent) - if (result != 0) { - println(coint.alpha()) - println(coint.beta()) - } - return result - } - - override fun generate(event: Event): List { - val contenders = mutableMapOf() - for ((asset, action) in event.prices) { - val priceSeries = history.getOrPut(asset) { PriceSeries(period) } - val price = action.getPrice(priceType) - if (priceSeries.add(price)) { - val data = priceSeries.toDoubleArray() - contenders[asset] = data - } - } - for ((asset1, data1) in contenders) { - for ((asset2, data2) in contenders) { - if (asset1 == asset2) continue - val fData = mutableListOf() - for (i in data1.indices) fData.add(doubleArrayOf(data1[i], data2[i])) - val r = johansenTest(fData.toTypedArray()) - if (r != 0) println("${asset1.symbol} ${asset2.symbol} $r") - } - - } - - return emptyList() - } - - } - - val strat = PairTradingStrategy(60) - val rq = Roboquant(strat) - rq.run(feed) - -} - - - -fun main() { - val feed = AvroFeed.sp500() - runJohansenTest(getTimeSeriesAppleTesla(feed)) - runJohansenTest(getTimeSeriesHighLow(feed)) - testStrat(feed) -} - - diff --git a/roboquant/src/main/kotlin/org/roboquant/brokers/Account.kt b/roboquant/src/main/kotlin/org/roboquant/brokers/Account.kt index 78e8aa85..c051a6d2 100644 --- a/roboquant/src/main/kotlin/org/roboquant/brokers/Account.kt +++ b/roboquant/src/main/kotlin/org/roboquant/brokers/Account.kt @@ -92,18 +92,6 @@ class Account( get() = positions.map { it.asset }.toSet() - @Suppress("unused") - fun contractValue(asset: Asset, size: Size, price: Double, time: Instant) : Double { - val newPrice = if (asset.currency != baseCurrency) { - val amount = Amount(asset.currency, price) - amount.convert(baseCurrency, time).value - } else { - price - } - - return size.toDouble() * newPrice * asset.multiplier - } - /** * Get the associated trades for the provided [orders]. If no orders are provided all [closedOrders] linked to this * account instance are used. diff --git a/roboquant/src/main/kotlin/org/roboquant/common/Asset.kt b/roboquant/src/main/kotlin/org/roboquant/common/Asset.kt index 97715c62..459f02e2 100644 --- a/roboquant/src/main/kotlin/org/roboquant/common/Asset.kt +++ b/roboquant/src/main/kotlin/org/roboquant/common/Asset.kt @@ -38,16 +38,14 @@ import java.time.format.DateTimeFormatter * @property type type of asset class, default is [AssetType.STOCK] * @property currency currency, default is [Currency.USD] * @property exchange Exchange this asset is traded on, default is [Exchange.DEFAULT] - * @property multiplier contract size multiplier, default is 1.0. * @property id asset identifier, default is an empty string * @constructor Create a new asset */ -data class Asset( +open class Asset( val symbol: String, val type: AssetType = AssetType.STOCK, val currency: Currency = Currency.USD, val exchange: Exchange = Exchange.DEFAULT, - val multiplier: Double = 1.0, val id: String = "" ) : Comparable { @@ -59,49 +57,23 @@ data class Asset( type: AssetType = AssetType.STOCK, currencyCode: String, exchangeCode: String = "", - multiplier: Double = 1.0, id: String = "" - ) : this(symbol, type, Currency.getInstance(currencyCode), Exchange.getInstance(exchangeCode), multiplier, id) + ) : this(symbol, type, Currency.getInstance(currencyCode), Exchange.getInstance(exchangeCode), id) init { require(symbol.isNotBlank()) { "Symbol in an asset cannot be empty or blank" } } + fun copy(symbol: String = this.symbol): Asset { + return Asset(symbol, this.type, this.currency, this.exchange, this.id) + } + /** * Contains methods to create specific asset types, like options or futures using international standards to * generate the appropriate symbol name. */ companion object { - /** - * Returns an option contract using the OCC (`Options Clearing Corporation`) option symbol standard. - * The OCC option symbol string consists of four parts: - * - * 1. Uppercase [symbol] of the underlying stock or ETF, padded with trailing spaces to six characters - * 2. The [expiration] date, in the format `yymmdd` - * 3. The Option [type], single character either P(ut) or C(all) - * 4. The strike price, as the [price] x 1000, front padded with zeros to make it eight digits - */ - fun optionContract( - symbol: String, - expiration: LocalDate, - type: Char, - price: BigDecimal, - multiplier: Double = 100.0, - currencyCode: String = "USD", - exchangeCode: String = "", - id: String = "" - ): Asset { - require(symbol.isNotBlank()) { "Symbol cannot be blank" } - require(type in setOf('P', 'C')) { "Type should be P or C" } - val formatter = DateTimeFormatter.ofPattern("yyMMdd") - val optionSymbol = "%-6s".format(symbol.uppercase()) + - expiration.format(formatter) + - type.uppercase() + - "%08d".format(price.multiply(BigDecimal(1000)).toInt()) - - return Asset(optionSymbol, AssetType.OPTION, currencyCode, exchangeCode, multiplier, id) - } /** * Returns a future contract based on the provided parameters. It will generate a [symbol] name using the @@ -113,14 +85,13 @@ data class Asset( year: Int, currencyCode: String = "USD", exchangeCode: String = "", - multiplier: Double = 1.0, id: String = "" ): Asset { val months = arrayOf('F', 'G', 'H', 'J', 'K', 'M', 'N', 'Q', 'U', 'V', 'X', 'Z') val monthEncoding = months[month.value - 1] val yearCode = year.toString() val futureSymbol = "$symbol$monthEncoding${yearCode.takeLast(2)}" - return Asset(futureSymbol, AssetType.FUTURES, currencyCode, exchangeCode, multiplier, id) + return Asset(futureSymbol, AssetType.FUTURES, currencyCode, exchangeCode, id) } /** @@ -153,12 +124,11 @@ data class Asset( } /** - * Return the value of the asset given the provided [size] and [price]. The calculation takes the [multiplier] - * of the asset into account. + * Return the value of the asset given the provided [size] and [price]. */ - fun value(size: Size, price: Double): Amount { + open fun value(size: Size, price: Double): Amount { // If size is zero, an unknown price (Double.NanN) is fine - return if (size.iszero) Amount(currency, 0.0) else Amount(currency, size.toDouble() * multiplier * price) + return if (size.iszero) Amount(currency, 0.0) else Amount(currency, size.toDouble() * price) } /** @@ -203,12 +173,64 @@ data class Asset( if (type != other.type) return false if (currency != other.currency) return false if (exchange != other.exchange) return false - if (multiplier != other.multiplier) return false return id == other.id } } +open class OptionContract( + symbol: String, + currency: Currency = Currency.USD, + exchange: Exchange = Exchange.DEFAULT, + val contractSize: Double = 100.0, + id: String = "" +) : Asset(symbol, AssetType.OPTION, currency, exchange, id) { + + override fun value(size: Size, price: Double): Amount { + return if (size.iszero) Amount(currency, 0.0) else Amount(currency, size.toDouble() * contractSize * price) + } + + companion object { + + /** + * Returns an option contract using the OCC (`Options Clearing Corporation`) option symbol standard. + * The OCC option symbol string consists of four parts: + * + * 1. Uppercase [symbol] of the underlying stock or ETF, padded with trailing spaces to six characters + * 2. The [expiration] date, in the format `yymmdd` + * 3. The Option [type], single character either P(ut) or C(all) + * 4. The strike price, as the [price] x 1000, front padded with zeros to make it eight digits + */ + fun from( + symbol: String, + expiration: LocalDate, + type: Char, + price: BigDecimal, + currencyCode: String = "USD", + exchangeCode: String = "", + contractSize: Double = 100.0, + id: String = "" + ): OptionContract { + require(symbol.isNotBlank()) { "Symbol cannot be blank" } + require(type in setOf('P', 'C')) { "Type should be P or C" } + val formatter = DateTimeFormatter.ofPattern("yyMMdd") + val optionSymbol = "%-6s".format(symbol.uppercase()) + + expiration.format(formatter) + + type.uppercase() + + "%08d".format(price.multiply(BigDecimal(1000)).toInt()) + + return OptionContract( + optionSymbol, + Currency.getInstance(currencyCode), + Exchange.getInstance(exchangeCode), + contractSize, + id + ) + } + } + +} + /** * Get an asset based on its [symbol] name. Will throw a NoSuchElementException if no asset is found. If there are * multiple assets with the same symbol, the first one will be returned. @@ -274,7 +296,7 @@ fun Collection.summary(name: String = "assets"): Summary { lines.add(listOf("symbol", "type", "ccy", "exchange", "multiplier", "id")) forEach { with(it) { - lines.add(listOf(symbol, type, currency.currencyCode, exchange.exchangeCode, multiplier, id)) + lines.add(listOf(symbol, type, currency.currencyCode, exchange.exchangeCode, id)) } } lines.summary(name) diff --git a/roboquant/src/main/kotlin/org/roboquant/feeds/Item.kt b/roboquant/src/main/kotlin/org/roboquant/feeds/Item.kt index 61bc8a79..0f885b63 100644 --- a/roboquant/src/main/kotlin/org/roboquant/feeds/Item.kt +++ b/roboquant/src/main/kotlin/org/roboquant/feeds/Item.kt @@ -73,6 +73,11 @@ interface PriceItem : Item { } +enum class PriceItemType { + BAR, QUOTE, TRADE, BOOK +} + + /** * Provides open, high, low, and close prices and volume for a single asset. If the volume is not available, it * will return Double.NaN instead. Often this type of price action is also referred to as a candlestick. diff --git a/roboquant/src/main/kotlin/org/roboquant/feeds/csv/CSVConfig.kt b/roboquant/src/main/kotlin/org/roboquant/feeds/csv/CSVConfig.kt index c0061ecc..49dc028d 100644 --- a/roboquant/src/main/kotlin/org/roboquant/feeds/csv/CSVConfig.kt +++ b/roboquant/src/main/kotlin/org/roboquant/feeds/csv/CSVConfig.kt @@ -226,8 +226,7 @@ data class CSVConfig( symbol = config.getOrDefault("symbol", "TEMPLATE"), type = AssetType.valueOf(config.getOrDefault("type", "STOCK")), currencyCode = config.getOrDefault("currency", "USD"), - exchangeCode = config.getOrDefault("exchange", ""), - multiplier = config.getOrDefault("multiplier", "1.0").toDouble() + exchangeCode = config.getOrDefault("exchange", "") ) } diff --git a/roboquant/src/main/kotlin/org/roboquant/feeds/random/RandomPriceGenerator.kt b/roboquant/src/main/kotlin/org/roboquant/feeds/random/RandomPriceGenerator.kt index eafea863..57bf1de2 100644 --- a/roboquant/src/main/kotlin/org/roboquant/feeds/random/RandomPriceGenerator.kt +++ b/roboquant/src/main/kotlin/org/roboquant/feeds/random/RandomPriceGenerator.kt @@ -19,9 +19,7 @@ package org.roboquant.feeds.random import org.roboquant.common.Asset import org.roboquant.common.Config import org.roboquant.common.TimeSpan -import org.roboquant.feeds.PriceBar -import org.roboquant.feeds.PriceItem -import org.roboquant.feeds.TradePrice +import org.roboquant.feeds.* import java.util.* internal class RandomPriceGenerator( @@ -29,7 +27,7 @@ internal class RandomPriceGenerator( private val priceChange: Double, private val volumeRange: Int, private val timeSpan: TimeSpan, - private val generateBars: Boolean, + private val priceType: PriceItemType, seed: Int ) { @@ -54,6 +52,12 @@ internal class RandomPriceGenerator( } } + private fun priceQupte(asset: Asset, price: Double): PriceItem { + val midPoint = price.nextPrice() + val volume = random.nextInt(volumeRange / 2, volumeRange * 2).toDouble() + return PriceQuote(asset, midPoint * 0.99, volume, midPoint * 1.01, volume) + } + /** * Generate random single price actions */ @@ -66,7 +70,13 @@ internal class RandomPriceGenerator( for ((idx, asset) in assets.withIndex()) { val lastPrice = prices[idx] val price = lastPrice.nextPrice().coerceAtLeast(priceChange * 2.0) - val action = if (generateBars) priceBar(asset, price) else tradePrice(asset, price) + val action = when (priceType) { + PriceItemType.BAR -> priceBar(asset, price) + PriceItemType.TRADE -> tradePrice(asset, price) + PriceItemType.QUOTE -> priceQupte(asset, price) + else -> throw UnsupportedOperationException("Unknown price type: $priceType") + } + add(action) prices[idx] = price } diff --git a/roboquant/src/main/kotlin/org/roboquant/feeds/random/RandomWalkFeed.kt b/roboquant/src/main/kotlin/org/roboquant/feeds/random/RandomWalkFeed.kt index 13a1aa62..735782a9 100644 --- a/roboquant/src/main/kotlin/org/roboquant/feeds/random/RandomWalkFeed.kt +++ b/roboquant/src/main/kotlin/org/roboquant/feeds/random/RandomWalkFeed.kt @@ -20,6 +20,7 @@ import org.roboquant.common.* import org.roboquant.feeds.Event import org.roboquant.feeds.EventChannel import org.roboquant.feeds.HistoricFeed +import org.roboquant.feeds.PriceItemType import java.time.Instant import java.time.LocalDate @@ -39,7 +40,7 @@ import java.time.LocalDate * @property timeframe the timeframe of this random walk * @property timeSpan the timeSpan between two events, default is `1.days` * @param nAssets the number of assets to generate, symbol names will be ASSET1, ASSET2, ..., ASSET. Default is 10. - * @property generateBars should PriceBars be generated or plain TradePrice, default is true + * @property priceType should PriceBars be generated or plain TradePrice, default is true * @property volumeRange what is the volume range, default = 1000 * @property priceChange the price range, the default is 10 bips. * @param template template to use when generating assets @@ -49,7 +50,7 @@ class RandomWalkFeed( override val timeframe: Timeframe, private val timeSpan: TimeSpan = 1.days, nAssets: Int = 10, - private val generateBars: Boolean = true, + private val priceType: PriceItemType = PriceItemType.BAR, private val volumeRange: Int = 1000, private val priceChange: Double = 10.bips, template: Asset = Asset("%s"), @@ -83,7 +84,7 @@ class RandomWalkFeed( * @see HistoricFeed.play */ override suspend fun play(channel: EventChannel) { - val gen = RandomPriceGenerator(assets.toList(), priceChange, volumeRange, timeSpan, generateBars, seed) + val gen = RandomPriceGenerator(assets.toList(), priceChange, volumeRange, timeSpan, priceType, seed) var time = timeframe.start while (timeframe.contains(time)) { val actions = gen.next() @@ -103,20 +104,20 @@ class RandomWalkFeed( /** * Create a random walk for the last [years], generating daily prices */ - fun lastYears(years: Int = 1, nAssets: Int = 10, generateBars: Boolean = true): RandomWalkFeed { + fun lastYears(years: Int = 1, nAssets: Int = 10, priceType: PriceItemType = PriceItemType.BAR): RandomWalkFeed { val lastYear = LocalDate.now().year val tf = Timeframe.fromYears(lastYear - years, lastYear) - return RandomWalkFeed(tf, 1.days, nAssets, generateBars) + return RandomWalkFeed(tf, 1.days, nAssets, priceType) } /** * Create a random walk for the last [days], generating minute prices. */ - fun lastDays(days: Int = 1, nAssets: Int = 10, generateBars: Boolean = true): RandomWalkFeed { + fun lastDays(days: Int = 1, nAssets: Int = 10, priceType: PriceItemType = PriceItemType.BAR): RandomWalkFeed { val last = Instant.now() val first = last - days.days val tf = Timeframe(first, last) - return RandomWalkFeed(tf, 1.minutes, nAssets, generateBars) + return RandomWalkFeed(tf, 1.minutes, nAssets, priceType) } } diff --git a/roboquant/src/main/kotlin/org/roboquant/feeds/random/RandomWalkLiveFeed.kt b/roboquant/src/main/kotlin/org/roboquant/feeds/random/RandomWalkLiveFeed.kt index f293d46d..83fcbb05 100644 --- a/roboquant/src/main/kotlin/org/roboquant/feeds/random/RandomWalkLiveFeed.kt +++ b/roboquant/src/main/kotlin/org/roboquant/feeds/random/RandomWalkLiveFeed.kt @@ -32,7 +32,7 @@ import java.time.Instant * * @property timeSpan the timeSpan between two events, default is `1.seconds` * @param nAssets the number of assets to generate, default is 10. - * @property generateBars should PriceBars be generated or plain TradePrice, default is true + * @property priceItemType should PriceBars be generated or plain TradePrice * @property volumeRange what is the volume range, default = 1000 * @property priceChange the price range, the default is 10 bips (0.1%) * @param template template to use when generating assets. The symbol name will be used as a template string @@ -41,7 +41,7 @@ import java.time.Instant class RandomWalkLiveFeed( private val timeSpan: TimeSpan = 1.seconds, nAssets: Int = 10, - private val generateBars: Boolean = true, + private val priceItemType: PriceItemType= PriceItemType.BAR, private val volumeRange: Int = 1000, private val priceChange: Double = 10.bips, template: Asset = Asset("%s"), @@ -63,7 +63,7 @@ class RandomWalkLiveFeed( * @see Feed.play */ override suspend fun play(channel: EventChannel) { - val gen = RandomPriceGenerator(assets.toList(), priceChange, volumeRange, timeSpan, generateBars, seed) + val gen = RandomPriceGenerator(assets.toList(), priceChange, volumeRange, timeSpan, priceItemType, seed) var time = Instant.now() while (true) { if (channel.closed) return diff --git a/roboquant/src/main/kotlin/org/roboquant/feeds/util/AssetSerializer.kt b/roboquant/src/main/kotlin/org/roboquant/feeds/util/AssetSerializer.kt index ddf314fa..c28ebd7c 100644 --- a/roboquant/src/main/kotlin/org/roboquant/feeds/util/AssetSerializer.kt +++ b/roboquant/src/main/kotlin/org/roboquant/feeds/util/AssetSerializer.kt @@ -35,8 +35,6 @@ object AssetSerializer { sb.append(SEP) if (exchange.exchangeCode != "") sb.append(exchange.exchangeCode) sb.append(SEP) - if (multiplier != 1.0) sb.append(multiplier) - sb.append(SEP) if (id.isNotEmpty()) sb.append(id) sb.append(SEP) @@ -64,8 +62,7 @@ object AssetSerializer { if (l > 1 && e[1].isNotEmpty()) AssetType.valueOf(e[1]) else AssetType.STOCK, if (l > 2 && e[2].isNotEmpty()) e[2] else "USD", if (l > 3) e[3] else "", - if (l > 4 && e[4].isNotEmpty()) e[4].toDouble() else 1.0, - if (l > 5) e[5] else "", + if (l > 4) e[4] else "", ) } diff --git a/roboquant/src/main/kotlin/org/roboquant/journals/MultiRunJournal.kt b/roboquant/src/main/kotlin/org/roboquant/journals/MultiRunJournal.kt index 804f518a..6271710a 100644 --- a/roboquant/src/main/kotlin/org/roboquant/journals/MultiRunJournal.kt +++ b/roboquant/src/main/kotlin/org/roboquant/journals/MultiRunJournal.kt @@ -14,10 +14,13 @@ class MultiRunJournal(private val fn: (String) -> MetricsJournal) { companion object { private var cnt = 0 + + @Synchronized + fun nextRun(): String = "run-${cnt++}" } @Synchronized - fun getJournal(run: String = "run-${cnt++}"): MetricsJournal { + fun getJournal(run: String = nextRun()): MetricsJournal { if (run !in journals) { val journal = fn(run) journals[run] = journal diff --git a/roboquant/src/test/kotlin/org/roboquant/common/AssetTest.kt b/roboquant/src/test/kotlin/org/roboquant/common/AssetTest.kt index a85dd451..91a372b8 100644 --- a/roboquant/src/test/kotlin/org/roboquant/common/AssetTest.kt +++ b/roboquant/src/test/kotlin/org/roboquant/common/AssetTest.kt @@ -39,6 +39,8 @@ internal class AssetTest { assertNotEquals(c, d) } + + @Test fun sorting() { val a = Asset("ABC") @@ -48,9 +50,15 @@ internal class AssetTest { } @Test - fun testAssetTypeConstructors() { - val a = Asset.optionContract("SPX", LocalDate.parse("2014-11-22"), 'P', BigDecimal("19.50")) + fun optionContract() { + val a = OptionContract.from("SPX", LocalDate.parse("2014-11-22"), 'P', BigDecimal("19.50")) assertEquals("SPX 141122P00019500", a.symbol) + assertEquals(100.0, a.contractSize) + } + + + @Test + fun testAssetTypeConstructors() { val b = Asset.futureContract("GC", Month.DECEMBER, 18) val b2 = Asset.futureContract("GC", Month.DECEMBER, 2018) @@ -99,21 +107,20 @@ internal class AssetTest { @Test fun contractValue() { - val a = Asset("ABC", multiplier = 100.0) - assertEquals(25000.0.USD, a.value(Size(10), 25.0)) + val a = Asset("ABC") + assertEquals(250.0.USD, a.value(Size(10), 25.0)) - val b = Asset("ABC") - assertEquals((-250.0).USD, b.value(Size(-10), 25.0)) + assertEquals((-250.0).USD, a.value(Size(-10), 25.0)) } @Test fun contractSize() { - val a = Asset("ABC", multiplier = 100.0) + val a = Asset("ABC") val s = a.contractSize(1000.0, 1.0) - assertEquals(Size(10), s) + assertEquals(Size(1000), s) val s2 = a.contractSize(1000.0, 1.0, 4) - assertEquals(Size(10), s2) + assertEquals(Size(1000), s2) // decimal fractions cannot be negative assertThrows { a.contractSize(250.0, 1.0, -1) } diff --git a/roboquant/src/test/kotlin/org/roboquant/feeds/AggregatorFeedTest.kt b/roboquant/src/test/kotlin/org/roboquant/feeds/AggregatorFeedTest.kt index 375b2867..0bfe2a1e 100644 --- a/roboquant/src/test/kotlin/org/roboquant/feeds/AggregatorFeedTest.kt +++ b/roboquant/src/test/kotlin/org/roboquant/feeds/AggregatorFeedTest.kt @@ -89,7 +89,7 @@ internal class AggregatorFeedTest { fun basic2() { // 5-seconds window with 1-millisecond resolution val timeframe = Timeframe.parse("2022-01-01T00:00:00Z", "2022-01-01T00:00:05Z") - val feed = RandomWalkFeed(timeframe, 1.millis, generateBars = false) + val feed = RandomWalkFeed(timeframe, 1.millis, priceType = PriceItemType.TRADE) val items1 = feed.toList() val aggFeed = AggregatorFeed(feed, 1.seconds) @@ -109,7 +109,7 @@ internal class AggregatorFeedTest { fun parallel() { // 5-seconds window with 1-millisecond resolution val timeframe = Timeframe.parse("2022-01-01T00:00:00Z", "2022-01-01T00:00:05Z") - val feed = RandomWalkFeed(timeframe, 1.millis, generateBars = false) + val feed = RandomWalkFeed(timeframe, 1.millis, priceType = PriceItemType.TRADE) val aggFeed = AggregatorFeed(feed, 1.seconds) val jobs = ParallelJobs() @@ -125,7 +125,7 @@ internal class AggregatorFeedTest { fun combined() { // 5-seconds window with 1-millisecond resolution val timeframe = Timeframe.parse("2022-01-01T00:00:00Z", "2022-01-01T00:00:05Z") - val rw = RandomWalkFeed(timeframe, 1.millis, generateBars = false) + val rw = RandomWalkFeed(timeframe, 1.millis, priceType = PriceItemType.TRADE) val items1 = rw.toList() val aggFeed1 = AggregatorFeed(rw, 1.seconds) diff --git a/roboquant/src/test/kotlin/org/roboquant/feeds/random/RandomWalkFeedTest.kt b/roboquant/src/test/kotlin/org/roboquant/feeds/random/RandomWalkFeedTest.kt index 6a949cdf..dba5d962 100644 --- a/roboquant/src/test/kotlin/org/roboquant/feeds/random/RandomWalkFeedTest.kt +++ b/roboquant/src/test/kotlin/org/roboquant/feeds/random/RandomWalkFeedTest.kt @@ -46,12 +46,12 @@ internal class RandomWalkFeedTest { @Test fun itemTypes() = runBlocking { - val feed = RandomWalkFeed.lastYears(generateBars = false) + val feed = RandomWalkFeed.lastYears(priceType = PriceItemType.TRADE) val event = play(feed).receive() assertTrue(event.items.first() is TradePrice) val tl = Timeframe.fromYears(2010, 2012) - val feed2 = RandomWalkFeed(tl, generateBars = true) + val feed2 = RandomWalkFeed(tl, priceType = PriceItemType.BAR) val item2 = play(feed2).receive() assertTrue(item2.items.first() is PriceBar) }