Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft: Threadpool reorganization #57

Open
wants to merge 449 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
449 commits
Select commit Hold shift + click to select a range
d165ff2
tests
elcritch Sep 21, 2023
bab5cb9
tests
elcritch Sep 21, 2023
6745c29
tests
elcritch Sep 21, 2023
25bb865
tests
elcritch Sep 21, 2023
b224c1c
cleanup
elcritch Sep 21, 2023
113f59c
don't store string
elcritch Sep 22, 2023
06cb6fc
rename to SQLiteBackend
elcritch Sep 22, 2023
8356f6c
re-add async frontend
elcritch Sep 22, 2023
4750ac6
implementing query
elcritch Sep 22, 2023
cbb38a5
trying to fixup query
elcritch Sep 22, 2023
3ae1c60
porting query tests
elcritch Sep 22, 2023
a339bb3
porting query tests
elcritch Sep 22, 2023
3d7c3d2
porting query tests
elcritch Sep 22, 2023
d6d5978
porting query tests
elcritch Sep 22, 2023
0971986
porting query tests
elcritch Sep 22, 2023
185f162
porting query tests
elcritch Sep 22, 2023
10ab970
changes
elcritch Sep 25, 2023
0a86a34
change to non-closure iterator
elcritch Sep 25, 2023
f8ba1e7
change to non-closure iterator
elcritch Sep 25, 2023
598f63c
change to non-closure iterator
elcritch Sep 25, 2023
7bb11c0
change to non-closure iterator
elcritch Sep 25, 2023
f0fc8ce
change to non-closure iterator
elcritch Sep 25, 2023
0efc7f6
change to non-closure iterator
elcritch Sep 25, 2023
85352e8
change to non-closure iterator
elcritch Sep 25, 2023
0336b93
change to non-closure iterator
elcritch Sep 26, 2023
92dcbf7
change to non-closure iterator
elcritch Sep 26, 2023
269aec6
change to generics
elcritch Sep 26, 2023
24ca5f9
change to generics
elcritch Sep 26, 2023
9fe9951
change to generics
elcritch Sep 26, 2023
dbfb54f
change to generics
elcritch Sep 26, 2023
04a30da
change to generics
elcritch Sep 26, 2023
f9e5992
query fixes
elcritch Sep 26, 2023
3b30cc8
query fixes
elcritch Sep 26, 2023
ca108f5
query fixes
elcritch Sep 26, 2023
ccd5d64
query fixes
elcritch Sep 26, 2023
1f23fe5
query fixes
elcritch Sep 26, 2023
62c9e7c
query fixes
elcritch Sep 26, 2023
ebed992
query fixes
elcritch Sep 26, 2023
77173a5
query fixes
elcritch Sep 26, 2023
a16cdaa
query fixes
elcritch Sep 26, 2023
2e2da52
query fixes
elcritch Sep 26, 2023
206751a
query fixes
elcritch Sep 26, 2023
2de2650
query fixes
elcritch Sep 26, 2023
d3eb55f
query fixes
elcritch Sep 26, 2023
289a844
query fixes
elcritch Sep 26, 2023
a637e66
query fixes
elcritch Sep 26, 2023
84cdc1d
query fixes
elcritch Sep 26, 2023
0c72ad8
query fixes
elcritch Sep 26, 2023
73ec70f
query fixes
elcritch Sep 26, 2023
2861eba
simple iterator
elcritch Sep 26, 2023
7710dd7
updates
elcritch Sep 26, 2023
be94266
updates
elcritch Sep 26, 2023
c25eb3a
updates
elcritch Sep 26, 2023
46a2912
updates
elcritch Sep 26, 2023
1a6065b
updates
elcritch Sep 26, 2023
37dbd1c
simplifying
elcritch Sep 26, 2023
2aa8cfa
simplifying
elcritch Sep 26, 2023
575d973
simplifying
elcritch Sep 26, 2023
77e53d2
simplifying
elcritch Sep 26, 2023
53eedd9
simplifying
elcritch Sep 26, 2023
8e573bd
simplifying
elcritch Sep 26, 2023
cbad3c1
simplifying
elcritch Sep 26, 2023
cdb55c0
simplifying
elcritch Sep 26, 2023
d8d6ad7
integrate setups
elcritch Sep 26, 2023
0926eaf
integrate setups
elcritch Sep 26, 2023
73df678
integrate setups
elcritch Sep 26, 2023
f159eff
integrate setups
elcritch Sep 26, 2023
8eef3e6
integrate setups
elcritch Sep 26, 2023
9c7c47e
integrate setups
elcritch Sep 26, 2023
0bd6bd5
integrate setups
elcritch Sep 26, 2023
fddbb99
await all
elcritch Sep 26, 2023
ca9edb2
await all
elcritch Sep 26, 2023
d34cbd7
integrate setups
elcritch Sep 26, 2023
b2de461
setup query
elcritch Sep 26, 2023
8595878
setup query
elcritch Sep 26, 2023
ca14322
setup query
elcritch Sep 26, 2023
7a0c660
setup query
elcritch Sep 26, 2023
7a4b9ac
setup query
elcritch Sep 26, 2023
86ee5d9
setup query
elcritch Sep 26, 2023
57e2a70
setup query
elcritch Sep 26, 2023
8699cfe
setup query
elcritch Sep 26, 2023
f8f0a72
setup query
elcritch Sep 26, 2023
2600d24
setup query end
elcritch Sep 26, 2023
33ca434
setup query end
elcritch Sep 26, 2023
173d426
setup query end
elcritch Sep 26, 2023
4be88df
setup query end
elcritch Sep 26, 2023
77a147c
setup query end
elcritch Sep 26, 2023
b6baefc
setup query end
elcritch Sep 26, 2023
eec66c9
setup query end
elcritch Sep 26, 2023
911de83
setup query end
elcritch Sep 26, 2023
9c6f2fc
setup query
elcritch Sep 26, 2023
5796d4d
setup query
elcritch Sep 26, 2023
abfd12f
setup query
elcritch Sep 26, 2023
892ec38
fixup databuffer
elcritch Sep 26, 2023
555f1fe
fixup databuffer
elcritch Sep 26, 2023
61e8fd4
fixup databuffer
elcritch Sep 26, 2023
6e9325e
testing
elcritch Sep 26, 2023
54a0412
testing
elcritch Sep 26, 2023
c31f189
testing
elcritch Sep 26, 2023
36e6679
fixup backend types
elcritch Sep 27, 2023
cfc743c
result types
elcritch Sep 27, 2023
e75081f
result types
elcritch Sep 27, 2023
c1f5d43
result types
elcritch Sep 27, 2023
da740f1
result types
elcritch Sep 27, 2023
b8dad8b
result types
elcritch Sep 27, 2023
1aec1a4
result types
elcritch Sep 27, 2023
d0ee284
result types
elcritch Sep 27, 2023
f59cf06
test query
elcritch Sep 27, 2023
b4ce286
test query
elcritch Sep 27, 2023
690b3d4
test query
elcritch Sep 27, 2023
8aa3618
test query
elcritch Sep 27, 2023
4b4403b
test query
elcritch Sep 27, 2023
a4c3574
test query
elcritch Sep 27, 2023
6c95c29
test query
elcritch Sep 27, 2023
4e63ca4
test query
elcritch Sep 27, 2023
d411b56
test query
elcritch Sep 27, 2023
24748cd
test query
elcritch Sep 27, 2023
be70ddb
test query
elcritch Sep 27, 2023
b746a26
test query
elcritch Sep 27, 2023
70b5956
test query
elcritch Sep 27, 2023
02de983
test query
elcritch Sep 27, 2023
0fedfcf
test query - it runs!
elcritch Sep 27, 2023
6b90743
test query - it runs!
elcritch Sep 27, 2023
f4d0705
test query - it runs!
elcritch Sep 27, 2023
9cd0ae7
test query - it runs!
elcritch Sep 27, 2023
f8446de
test query - it runs!
elcritch Sep 27, 2023
317d42c
test query - it runs!
elcritch Sep 27, 2023
dacc28a
fix occasional deadlock
elcritch Sep 27, 2023
5c2cc57
fix occasional deadlock
elcritch Sep 27, 2023
4ff162e
fix occasional deadlock
elcritch Sep 27, 2023
7169a7d
fix occasional deadlock
elcritch Sep 27, 2023
2d84385
cleanup
elcritch Sep 27, 2023
55b118c
cleanup
elcritch Sep 27, 2023
2255ea1
Revert "cleanup"
elcritch Sep 27, 2023
c67b61a
cleanup
elcritch Sep 27, 2023
e2b6005
cleanup
elcritch Sep 27, 2023
76590d4
cleanup
elcritch Sep 27, 2023
c7284b3
cleanup
elcritch Sep 27, 2023
bf360f4
Merge branch 'master' into threadpool-refactor
elcritch Sep 27, 2023
686ea8e
cleanup
elcritch Sep 27, 2023
25e16f9
docs
elcritch Sep 27, 2023
c168c8e
docs
elcritch Sep 27, 2023
064a585
Merge branch 'threadpool-refactor' of github.com:codex-storage/nim-da…
elcritch Sep 27, 2023
10bee68
docs
elcritch Sep 27, 2023
d1f503f
reduce thread count
elcritch Sep 27, 2023
51d2b47
initial take at making fsds synchronous
elcritch Sep 27, 2023
24ac1a8
initial take at making fsds synchronous
elcritch Sep 27, 2023
653ec99
initial take at making fsds synchronous
elcritch Sep 27, 2023
5242b85
initial take at making fsds synchronous
elcritch Sep 27, 2023
bb9e343
initial take at making fsds synchronous
elcritch Sep 27, 2023
235232a
initial take at making fsds synchronous
elcritch Sep 27, 2023
5e42426
rework tuple types
elcritch Sep 27, 2023
49d846f
make Threadproxyds generic
elcritch Sep 27, 2023
6a4e460
queryLocks can be done at FsDs backend level
elcritch Sep 27, 2023
b4b534b
loop tests
elcritch Sep 27, 2023
0300841
loop tests
elcritch Sep 27, 2023
a4f0c48
fix merge
elcritch Sep 27, 2023
1fd80c6
fix merge
elcritch Sep 27, 2023
c51d354
add nextSignal using mutex
elcritch Sep 27, 2023
30728c1
add nextSignal using mutex
elcritch Sep 27, 2023
e571d21
add nextSignal using mutex
elcritch Sep 27, 2023
3cc21b3
add nextSignal using mutex
elcritch Sep 27, 2023
1da59ba
add nextSignal using mutex
elcritch Sep 27, 2023
ac77917
remove ctx locks
elcritch Sep 27, 2023
ca5695f
remove ctx locks
elcritch Sep 27, 2023
fbc0061
remove ctx locks
elcritch Sep 27, 2023
efd2e1d
running 1000+ outer loops
elcritch Sep 27, 2023
a4748ef
running 1000+ outer loops
elcritch Sep 27, 2023
78ea3b1
global taskpool
elcritch Sep 27, 2023
5afec2b
change nextSignal back to ThreadSignalPtr for timeouts
elcritch Sep 27, 2023
6952987
running 1000+ outer loops
elcritch Sep 27, 2023
74953e1
try newer questionable?
elcritch Sep 27, 2023
8db2857
Merge branch 'threadpool-refactor-generics' into threadpool-refactor-…
elcritch Sep 27, 2023
9fdba84
try newer questionable?
elcritch Sep 28, 2023
10bad7e
try newer questionable?
elcritch Sep 28, 2023
2eb0ee6
update fsds
elcritch Sep 28, 2023
1317120
update fsds
elcritch Sep 28, 2023
bda61df
update fsds
elcritch Sep 28, 2023
36ec858
refactor tests
elcritch Sep 28, 2023
439fd92
refactor tests
elcritch Sep 28, 2023
c215f9c
refactor tests
elcritch Sep 28, 2023
ef5a30f
refactor tests
elcritch Sep 28, 2023
ed34fe2
refactor tests
elcritch Sep 28, 2023
2a96b1b
refactor tests
elcritch Sep 28, 2023
4fa532e
refactor tests
elcritch Sep 28, 2023
1b7866a
refactor
elcritch Sep 28, 2023
f587677
refactor - tests
elcritch Sep 28, 2023
9e946e6
refactor - tests
elcritch Sep 28, 2023
1c2c5f1
refactor - tests
elcritch Sep 28, 2023
5345550
refactor - tests
elcritch Sep 28, 2023
e778fdb
refactor - tests
elcritch Sep 28, 2023
d5850eb
refactor - tests
elcritch Sep 28, 2023
ade0898
refactor - tests
elcritch Sep 28, 2023
cac5d52
refactor - tests
elcritch Sep 28, 2023
d4a7549
refactor - tests
elcritch Sep 28, 2023
17337ea
refactor - tests
elcritch Sep 28, 2023
f0591a0
refactor - tests
elcritch Sep 28, 2023
16bd98d
refactor - tests
elcritch Sep 28, 2023
ae31185
refactor - tests
elcritch Sep 28, 2023
dcb70ca
refactor - tests
elcritch Sep 28, 2023
0f0e113
refactor - tests
elcritch Sep 28, 2023
fb73d6b
refactor - tests
elcritch Sep 28, 2023
1ef4825
refactor - tests
elcritch Sep 28, 2023
ea4790a
refactor - tests
elcritch Sep 28, 2023
58fcdd9
switch to two sqlites for now
elcritch Sep 28, 2023
e761957
switch to two sqlites for now
elcritch Sep 28, 2023
b7c9308
switch to two sqlites for now
elcritch Sep 28, 2023
8f2f06e
skip fsds for now
elcritch Sep 28, 2023
4b804d1
change threding lib
elcritch Sep 28, 2023
e6afc68
whats up with windows + questionable + generics
elcritch Sep 28, 2023
eba4033
whats up with windows + questionable + generics
elcritch Sep 28, 2023
e653364
whats up with windows + questionable + generics
elcritch Sep 28, 2023
f98432d
fixed thing
elcritch Sep 28, 2023
d2d051d
fixed thing
elcritch Sep 28, 2023
3822a97
fixed thing
elcritch Sep 28, 2023
46f22dc
test queries
elcritch Sep 28, 2023
bd79cb7
test task cancel
elcritch Sep 28, 2023
2481746
test task cancel
elcritch Sep 28, 2023
43520c3
test task cancel
elcritch Sep 28, 2023
47fd3ba
test task cancel
elcritch Sep 28, 2023
0df63d6
test task cancel
elcritch Sep 28, 2023
374e5a8
test task cancel
elcritch Sep 28, 2023
4002ba9
reorg
elcritch Sep 29, 2023
01ccbb5
reorg
elcritch Sep 29, 2023
82bf1f1
reorg
elcritch Sep 29, 2023
3e477f3
reorg
elcritch Sep 29, 2023
f82ea14
reorg
elcritch Sep 29, 2023
96c54f6
reog
elcritch Sep 29, 2023
4739167
reorg
elcritch Sep 29, 2023
8ce566a
reorg
elcritch Sep 29, 2023
5f84ed4
reorg
elcritch Sep 29, 2023
d24d3e5
reorg
elcritch Sep 29, 2023
e945e30
reorg
elcritch Sep 29, 2023
61c14e4
reorg
elcritch Sep 29, 2023
de20fa4
reorg
elcritch Sep 29, 2023
9cc20e9
reorg
elcritch Sep 29, 2023
2a4d96e
reorg
elcritch Sep 29, 2023
ebc93e8
add files lock
elcritch Sep 29, 2023
c85cf18
add files lock
elcritch Sep 29, 2023
9184077
add files lock
elcritch Sep 29, 2023
61d01f8
add files lock
elcritch Sep 29, 2023
7d98f5e
switch back to upstream threading lib
elcritch Oct 3, 2023
a626a03
cleanup
elcritch Oct 20, 2023
235bddb
add compile option to use old sync setup
elcritch Oct 20, 2023
a8da4c4
copy paste sync-sqlite setup too
elcritch Oct 20, 2023
4844f7d
updates
elcritch Oct 20, 2023
ab2e7b5
updates
elcritch Oct 20, 2023
e7aef8e
updates
elcritch Oct 20, 2023
5dcd8ce
run and build sync tests
elcritch Oct 20, 2023
4815796
rename toSeq to toSequence to avoid conflicting with system.toSeq
elcritch Oct 23, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -134,3 +134,10 @@ jobs:
nim --version
echo
nimble --verbose test

- name: Build and run tests
run: |
source "${HOME}/.bash_env"
nim --version
echo
nimble --verbose -d:datastoreUseAsync=false test
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,6 @@ coverage
datastore.nims
nimcache
TODO
nim.cfg
nimble.develop
nimble.paths
6 changes: 6 additions & 0 deletions config.nims
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,16 @@
--styleCheck:usages
--styleCheck:error

# --d:"datastoreUseAsync=false"

when (NimMajor, NimMinor) == (1, 2):
switch("hint", "Processing:off")
switch("hint", "XDeclaredButNotUsed:off")
switch("warning", "ObservableStores:off")

when (NimMajor, NimMinor) > (1, 2):
switch("hint", "XCannotRaiseY:off")
# begin Nimble config (version 2)
when withDir(thisDir(), system.fileExists("nimble.paths")):
include "nimble.paths"
# end Nimble config
2 changes: 1 addition & 1 deletion datastore.nim
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@ import ./datastore/sql
import ./datastore/mountedds
import ./datastore/tieredds

export datastore, fsds, mountedds, tieredds, sql
export datastore, fsds, sql, mountedds, tieredds
12 changes: 8 additions & 4 deletions datastore.nimble
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,18 @@ author = "Status Research & Development GmbH"
description = "Simple, unified API for multiple data stores"
license = "Apache License 2.0 or MIT"

requires "nim >= 1.2.0",
requires "nim >= 1.6.14",
"asynctest >= 0.3.1 & < 0.4.0",
"chronos",
"questionable >= 0.10.3 & < 0.11.0",
"chronos#0277b65be2c7a365ac13df002fba6e172be55537",
"questionable#head",
"sqlite3_abi",
"stew",
"unittest2",
"upraises >= 0.1.0 & < 0.2.0"
"pretty",
"threading#7c033b9135b4baae84e6c89af7548848bba7dc02", # or more recent
"taskpools",
"upraises >= 0.1.0 & < 0.2.0",
"chronicles"

task coverage, "generates code coverage report":
var (output, exitCode) = gorgeEx("which lcov")
Expand Down
29 changes: 18 additions & 11 deletions datastore/datastore.nim
Original file line number Diff line number Diff line change
Expand Up @@ -8,37 +8,44 @@ import ./types

export key, query, types

const datastoreUseAsync* {.booldefine.} = true

push: {.upraises: [].}

type
BatchEntry* = tuple[key: Key, data: seq[byte]]

method has*(self: Datastore, key: Key): Future[?!bool] {.base, locks: "unknown".} =
method has*(self: Datastore, key: Key): Future[?!bool] {.base, locks: "unknown", raises: [].} =
raiseAssert("Not implemented!")

method delete*(self: Datastore, key: Key): Future[?!void] {.base, locks: "unknown".} =
method delete*(self: Datastore, key: Key): Future[?!void] {.base, locks: "unknown", raises: [].} =
raiseAssert("Not implemented!")

method delete*(self: Datastore, keys: seq[Key]): Future[?!void] {.base, locks: "unknown".} =
method delete*(self: Datastore, keys: seq[Key]): Future[?!void] {.base, locks: "unknown", raises: [].} =
raiseAssert("Not implemented!")

method get*(self: Datastore, key: Key): Future[?!seq[byte]] {.base, locks: "unknown".} =
method get*(self: Datastore, key: Key): Future[?!seq[byte]] {.base, locks: "unknown", raises: [].} =
raiseAssert("Not implemented!")

method put*(self: Datastore, key: Key, data: seq[byte]): Future[?!void] {.base, locks: "unknown".} =
method put*(self: Datastore, key: Key, data: seq[byte]): Future[?!void] {.base, locks: "unknown", raises: [].} =
raiseAssert("Not implemented!")

method put*(self: Datastore, batch: seq[BatchEntry]): Future[?!void] {.base, locks: "unknown".} =
method put*(self: Datastore, batch: seq[BatchEntry]): Future[?!void] {.base, locks: "unknown", raises: [].} =
raiseAssert("Not implemented!")

method close*(self: Datastore): Future[?!void] {.base, async, locks: "unknown".} =
method close*(self: Datastore): Future[?!void] {.base, locks: "unknown", raises: [].} =
raiseAssert("Not implemented!")

method query*(
self: Datastore,
query: Query): Future[?!QueryIter] {.base, gcsafe.} =
method query*(self: Datastore,
query: Query
): Future[?!QueryIter] {.base, gcsafe, raises: [].} =

raiseAssert("Not implemented!")

method queryIter*(self: Datastore,
query: Query
): ?!(iterator(): ?!QueryResponse) {.base, gcsafe, raises: [].} =
raiseAssert("Not implemented!")

proc contains*(self: Datastore, key: Key): Future[bool] {.async.} =
proc contains*(self: Datastore, key: Key): Future[bool] {.async, raises: [].} =
return (await self.has(key)) |? false
Loading