From 681860dde287f084aba3c1ce9a8f15f4007e484d Mon Sep 17 00:00:00 2001 From: natzcam Date: Thu, 2 Jan 2025 04:20:42 +0800 Subject: [PATCH 01/18] happy path --- .gitignore | 1 + index.js | 2 +- package.json | 6 ++++- test/blobs.js | 26 +++++++++++++++++++++ test/drives.js | 18 +++++++++++++++ test/helpers/index.js | 54 +++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 105 insertions(+), 2 deletions(-) create mode 100644 test/blobs.js create mode 100644 test/drives.js create mode 100644 test/helpers/index.js diff --git a/.gitignore b/.gitignore index 827c878..4ba0313 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ sandbox.js sandbox coverage package-lock.json +.vscode diff --git a/index.js b/index.js index e02007b..35e868d 100644 --- a/index.js +++ b/index.js @@ -273,7 +273,7 @@ module.exports = class ServeBlobs { } = opts if (!blob && !filename) { - throw new Error('Must specific a filename or blob') + throw new Error('Must specify a filename or blob') } const p = (protocol === 'http' && port === 80) diff --git a/package.json b/package.json index a073912..5e8ca02 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ } }, "scripts": { - "test": "standard" + "test": "standard && brittle test/*.js" }, "dependencies": { "bare-http1": "^4.0.2", @@ -30,6 +30,10 @@ "z32": "^1.1.0" }, "devDependencies": { + "brittle": "^3.7.0", + "corestore": "^6.18.4", + "hyperdrive": "^11.13.3", + "random-access-memory": "^6.2.1", "standard": "^17.1.2" }, "repository": { diff --git a/test/blobs.js b/test/blobs.js new file mode 100644 index 0000000..4a4cae2 --- /dev/null +++ b/test/blobs.js @@ -0,0 +1,26 @@ +const test = require('brittle') +const RAM = require('random-access-memory') +const Corestore = require('corestore') +const { tmpServeBlobs, request } = require('./helpers') + +test('can get blob from hypercore', async function (t) { + const store = new Corestore(RAM) + + const core = store.get({ name: 'test' }) + + await core.append([Buffer.from('Hello'), Buffer.from('World')]) + + const serve = tmpServeBlobs(t, store) + await serve.listen() + + const res = await request(serve, core.key, { + blob: { + blockOffset: 0, + blockLength: 2, + byteOffset: 2, + byteLength: 9 + } + }) + t.is(res.status, 200) + t.is(res.data, 'HelloWorl') +}) diff --git a/test/drives.js b/test/drives.js new file mode 100644 index 0000000..6b02661 --- /dev/null +++ b/test/drives.js @@ -0,0 +1,18 @@ +const test = require('brittle') +const RAM = require('random-access-memory') +const Corestore = require('corestore') +const { tmpHyperdrive, tmpServeBlobs, request } = require('./helpers') + +test('can get file from hyperdrive', async function (t) { + const store = new Corestore(RAM) + + const drive = tmpHyperdrive(t, store) + await drive.put('/file.txt', 'Here') + + const serve = tmpServeBlobs(t, store) + await serve.listen() + + const res = await request(serve, drive.key, { filename: '/file.txt' }) + t.is(res.status, 200) + t.is(res.data, 'Here') +}) diff --git a/test/helpers/index.js b/test/helpers/index.js new file mode 100644 index 0000000..0e35324 --- /dev/null +++ b/test/helpers/index.js @@ -0,0 +1,54 @@ +const http = require('http') +const ServeBlobs = require('../../index.js') +const Hyperdrive = require('hyperdrive') + +module.exports = { + request, + tmpServeBlobs, + tmpHyperdrive +} + +function get (link) { + return new Promise((resolve, reject) => { + const req = http.get(link, { + headers: { + Connection: 'close' + } + }) + + req.on('error', reject) + req.on('response', function (res) { + if (res.statusCode === 307) { + // follow redirect + get(new URL(link).origin + res.headers.location).then(resolve).catch(reject) + } else { + let buf = '' + res.setEncoding('utf-8') + res.on('data', function (data) { + buf += data + }) + res.on('end', function () { + resolve({ status: res.statusCode, data: buf }) + }) + } + }) + }) +} + +async function request (serve, key, opts) { + const link = serve.getLink(key, opts) + + return get(link) +} + +function tmpServeBlobs (t, store, opts) { + const serve = new ServeBlobs(store, opts) + t.teardown(() => serve.close()) + return serve +} + +function tmpHyperdrive (t, store) { + const drive = new Hyperdrive(store) + t.teardown(() => drive.close()) + return drive +} From ffa8dccd116fa4a3d74ab69208389c773b6f7cd9 Mon Sep 17 00:00:00 2001 From: natzcam Date: Thu, 2 Jan 2025 04:32:46 +0800 Subject: [PATCH 02/18] rename helpers --- test/blobs.js | 4 ++-- test/drives.js | 6 +++--- test/helpers/index.js | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/test/blobs.js b/test/blobs.js index 4a4cae2..62b8d27 100644 --- a/test/blobs.js +++ b/test/blobs.js @@ -1,7 +1,7 @@ const test = require('brittle') const RAM = require('random-access-memory') const Corestore = require('corestore') -const { tmpServeBlobs, request } = require('./helpers') +const { testServeBlobs, request } = require('./helpers') test('can get blob from hypercore', async function (t) { const store = new Corestore(RAM) @@ -10,7 +10,7 @@ test('can get blob from hypercore', async function (t) { await core.append([Buffer.from('Hello'), Buffer.from('World')]) - const serve = tmpServeBlobs(t, store) + const serve = testServeBlobs(t, store) await serve.listen() const res = await request(serve, core.key, { diff --git a/test/drives.js b/test/drives.js index 6b02661..594c2b3 100644 --- a/test/drives.js +++ b/test/drives.js @@ -1,15 +1,15 @@ const test = require('brittle') const RAM = require('random-access-memory') const Corestore = require('corestore') -const { tmpHyperdrive, tmpServeBlobs, request } = require('./helpers') +const { testHyperdrive, testServeBlobs, request } = require('./helpers') test('can get file from hyperdrive', async function (t) { const store = new Corestore(RAM) - const drive = tmpHyperdrive(t, store) + const drive = testHyperdrive(t, store) await drive.put('/file.txt', 'Here') - const serve = tmpServeBlobs(t, store) + const serve = testServeBlobs(t, store) await serve.listen() const res = await request(serve, drive.key, { filename: '/file.txt' }) diff --git a/test/helpers/index.js b/test/helpers/index.js index 0e35324..49b18e5 100644 --- a/test/helpers/index.js +++ b/test/helpers/index.js @@ -4,8 +4,8 @@ const Hyperdrive = require('hyperdrive') module.exports = { request, - tmpServeBlobs, - tmpHyperdrive + testServeBlobs, + testHyperdrive } function get (link) { @@ -41,13 +41,13 @@ async function request (serve, key, opts) { return get(link) } -function tmpServeBlobs (t, store, opts) { +function testServeBlobs (t, store, opts) { const serve = new ServeBlobs(store, opts) t.teardown(() => serve.close()) return serve } -function tmpHyperdrive (t, store) { +function testHyperdrive (t, store) { const drive = new Hyperdrive(store) t.teardown(() => drive.close()) return drive From 1785e320a76ac6e9009eb78ac8c4472601b9f65e Mon Sep 17 00:00:00 2001 From: natzcam Date: Thu, 2 Jan 2025 17:24:07 +0800 Subject: [PATCH 03/18] range tests --- package.json | 1 + test/blobs.js | 80 ++++++++++++++++++++++++++++++++++++------- test/helpers/index.js | 16 +++++++-- 3 files changed, 82 insertions(+), 15 deletions(-) diff --git a/package.json b/package.json index 5e8ca02..fd2f35d 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "devDependencies": { "brittle": "^3.7.0", "corestore": "^6.18.4", + "hyperblobs": "^2.7.4", "hyperdrive": "^11.13.3", "random-access-memory": "^6.2.1", "standard": "^17.1.2" diff --git a/test/blobs.js b/test/blobs.js index 62b8d27..bfff5e1 100644 --- a/test/blobs.js +++ b/test/blobs.js @@ -1,26 +1,82 @@ const test = require('brittle') const RAM = require('random-access-memory') const Corestore = require('corestore') -const { testServeBlobs, request } = require('./helpers') +const { testServeBlobs, request, testHyperblobs } = require('./helpers') test('can get blob from hypercore', async function (t) { const store = new Corestore(RAM) - const core = store.get({ name: 'test' }) + const blobs = testHyperblobs(t, store) - await core.append([Buffer.from('Hello'), Buffer.from('World')]) + const id = await blobs.put(Buffer.from('Hello World')) const serve = testServeBlobs(t, store) await serve.listen() - const res = await request(serve, core.key, { - blob: { - blockOffset: 0, - blockLength: 2, - byteOffset: 2, - byteLength: 9 - } - }) + const res = await request(serve, blobs.core.key, { blob: id }) + + t.is(res.status, 200) + t.is(res.data, 'Hello World') +}) + +test('can get blob from hypercore - multiple blocks', async function (t) { + const store = new Corestore(RAM) + + const blobs = testHyperblobs(t, store) + blobs.blockSize = 4 // force multiple blocks + + const id = await blobs.put(Buffer.from('Hello World')) + + const serve = testServeBlobs(t, store) + await serve.listen() + + const res = await request(serve, blobs.core.key, { blob: id }) + t.is(res.status, 200) - t.is(res.data, 'HelloWorl') + t.is(res.data, 'Hello World') +}) + +test('can get a partial blob from hypercore', async function (t) { + const store = new Corestore(RAM) + + const blobs = testHyperblobs(t, store) + + const id = await blobs.put(Buffer.from('Hello World')) + + const serve = testServeBlobs(t, store) + await serve.listen() + + const res = await request(serve, blobs.core.key, { blob: id, range: 'bytes=3-7' }) + t.is(res.status, 206) + t.is(res.data, 'lo Wo') +}) + +test('can get a partial blob from hypercore, but request the whole data', async function (t) { + const store = new Corestore(RAM) + + const blobs = testHyperblobs(t, store) + + const id = await blobs.put(Buffer.from('Hello World')) + + const serve = testServeBlobs(t, store) + await serve.listen() + + const res = await request(serve, blobs.core.key, { blob: id, range: 'bytes=0-10' }) + t.is(res.status, 206) + t.is(res.data, 'Hello World') +}) + +test('can get a partial blob from hypercore, out of range', async function (t) { + const store = new Corestore(RAM) + + const blobs = testHyperblobs(t, store) + + const id = await blobs.put(Buffer.from('Hello World')) + + const serve = testServeBlobs(t, store) + await serve.listen() + + const res = await request(serve, blobs.core.key, { blob: id, range: 'bytes=0-11' }) + t.is(res.status, 206) + t.is(res.data, 'Hello World') }) diff --git a/test/helpers/index.js b/test/helpers/index.js index 49b18e5..a004d37 100644 --- a/test/helpers/index.js +++ b/test/helpers/index.js @@ -1,18 +1,21 @@ const http = require('http') const ServeBlobs = require('../../index.js') const Hyperdrive = require('hyperdrive') +const Hyperblobs = require('hyperblobs') module.exports = { request, testServeBlobs, + testHyperblobs, testHyperdrive } -function get (link) { +function get (link, range = null) { return new Promise((resolve, reject) => { const req = http.get(link, { headers: { - Connection: 'close' + Connection: 'close', + range } }) @@ -38,7 +41,7 @@ function get (link) { async function request (serve, key, opts) { const link = serve.getLink(key, opts) - return get(link) + return get(link, opts.range) } function testServeBlobs (t, store, opts) { @@ -47,6 +50,13 @@ function testServeBlobs (t, store, opts) { return serve } +function testHyperblobs (t, store) { + const core = store.get({ name: 'test' }) + const blobs = new Hyperblobs(core) + t.teardown(() => blobs.close()) + return blobs +} + function testHyperdrive (t, store) { const drive = new Hyperdrive(store) t.teardown(() => drive.close()) From 4fcf2c806a96351922d3e09cd2d94c9a60e4b532 Mon Sep 17 00:00:00 2001 From: natzcam Date: Thu, 2 Jan 2025 17:35:58 +0800 Subject: [PATCH 04/18] default to byteLength --- index.js | 2 +- test/blobs.js | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 35e868d..65c7079 100644 --- a/index.js +++ b/index.js @@ -138,7 +138,7 @@ module.exports = class ServeBlobs { const end = blob.range.end === -1 ? blob.id.byteLength - 1 : blob.range.end start = blob.range.start - length = end - start + 1 + length = Math.min(blob.id.byteLength, end - start + 1) res.statusCode = 206 res.setHeader('Content-Range', 'bytes ' + start + '-' + end + '/' + blob.id.byteLength) diff --git a/test/blobs.js b/test/blobs.js index bfff5e1..3e10061 100644 --- a/test/blobs.js +++ b/test/blobs.js @@ -26,6 +26,7 @@ test('can get blob from hypercore - multiple blocks', async function (t) { blobs.blockSize = 4 // force multiple blocks const id = await blobs.put(Buffer.from('Hello World')) + t.is(id.blockLength, 3) // 3 blocks const serve = testServeBlobs(t, store) await serve.listen() From ebac8ecc3bc850a2f2a376342719a0924625c398 Mon Sep 17 00:00:00 2001 From: natzcam Date: Thu, 2 Jan 2025 17:45:20 +0800 Subject: [PATCH 05/18] more tests --- test/blobs.js | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/test/blobs.js b/test/blobs.js index 3e10061..3eb599d 100644 --- a/test/blobs.js +++ b/test/blobs.js @@ -67,7 +67,7 @@ test('can get a partial blob from hypercore, but request the whole data', async t.is(res.data, 'Hello World') }) -test('can get a partial blob from hypercore, out of range', async function (t) { +test('handle out of range header end', async function (t) { const store = new Corestore(RAM) const blobs = testHyperblobs(t, store) @@ -81,3 +81,33 @@ test('can get a partial blob from hypercore, out of range', async function (t) { t.is(res.status, 206) t.is(res.data, 'Hello World') }) + +test.solo('handle range header without end', async function (t) { + const store = new Corestore(RAM) + + const blobs = testHyperblobs(t, store) + + const id = await blobs.put(Buffer.from('Hello World')) + + const serve = testServeBlobs(t, store) + await serve.listen() + + const res = await request(serve, blobs.core.key, { blob: id, range: 'bytes=2-' }) + t.is(res.status, 206) + t.is(res.data, 'llo World') +}) + +test('handle invalid range header', async function (t) { + const store = new Corestore(RAM) + + const blobs = testHyperblobs(t, store) + + const id = await blobs.put(Buffer.from('Hello World')) + + const serve = testServeBlobs(t, store) + await serve.listen() + + const res = await request(serve, blobs.core.key, { blob: id, range: 'testing' }) + t.is(res.status, 200) + t.is(res.data, 'Hello World') +}) From 47fa59086b98508bb5accb6045e0c50370af8e34 Mon Sep 17 00:00:00 2001 From: natzcam Date: Thu, 2 Jan 2025 17:46:16 +0800 Subject: [PATCH 06/18] remove solo --- test/blobs.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/blobs.js b/test/blobs.js index 3eb599d..da6492c 100644 --- a/test/blobs.js +++ b/test/blobs.js @@ -82,7 +82,7 @@ test('handle out of range header end', async function (t) { t.is(res.data, 'Hello World') }) -test.solo('handle range header without end', async function (t) { +test('handle range header without end', async function (t) { const store = new Corestore(RAM) const blobs = testHyperblobs(t, store) From c3d3e5fc846910a83d554df99c3a0bd520b7e0af Mon Sep 17 00:00:00 2001 From: natzcam Date: Thu, 2 Jan 2025 17:52:00 +0800 Subject: [PATCH 07/18] demo --- index.js | 3 ++- test/blobs.js | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 65c7079..9723abe 100644 --- a/index.js +++ b/index.js @@ -138,7 +138,8 @@ module.exports = class ServeBlobs { const end = blob.range.end === -1 ? blob.id.byteLength - 1 : blob.range.end start = blob.range.start - length = Math.min(blob.id.byteLength, end - start + 1) + length = end - start + 1 + // length = Math.min(blob.id.byteLength, end - start + 1) res.statusCode = 206 res.setHeader('Content-Range', 'bytes ' + start + '-' + end + '/' + blob.id.byteLength) diff --git a/test/blobs.js b/test/blobs.js index da6492c..e785c17 100644 --- a/test/blobs.js +++ b/test/blobs.js @@ -67,7 +67,7 @@ test('can get a partial blob from hypercore, but request the whole data', async t.is(res.data, 'Hello World') }) -test('handle out of range header end', async function (t) { +test.solo('handle out of range header end', async function (t) { const store = new Corestore(RAM) const blobs = testHyperblobs(t, store) From 91a7c0cfba174367da697a9bc91b4e3282d67539 Mon Sep 17 00:00:00 2001 From: natzcam Date: Thu, 2 Jan 2025 17:54:21 +0800 Subject: [PATCH 08/18] make it more obvios --- test/blobs.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/blobs.js b/test/blobs.js index e785c17..c347400 100644 --- a/test/blobs.js +++ b/test/blobs.js @@ -77,7 +77,7 @@ test.solo('handle out of range header end', async function (t) { const serve = testServeBlobs(t, store) await serve.listen() - const res = await request(serve, blobs.core.key, { blob: id, range: 'bytes=0-11' }) + const res = await request(serve, blobs.core.key, { blob: id, range: 'bytes=0-20' }) t.is(res.status, 206) t.is(res.data, 'Hello World') }) From 301d764e3ec560b6b1347fcad925d016e0458f82 Mon Sep 17 00:00:00 2001 From: natzcam Date: Thu, 2 Jan 2025 18:30:58 +0800 Subject: [PATCH 09/18] handle close --- index.js | 1 - package.json | 2 +- test/helpers/index.js | 6 ++++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 9723abe..35e868d 100644 --- a/index.js +++ b/index.js @@ -139,7 +139,6 @@ module.exports = class ServeBlobs { start = blob.range.start length = end - start + 1 - // length = Math.min(blob.id.byteLength, end - start + 1) res.statusCode = 206 res.setHeader('Content-Range', 'bytes ' + start + '-' + end + '/' + blob.id.byteLength) diff --git a/package.json b/package.json index fd2f35d..87ec02d 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "get-mime-type": "^1.0.2", "hyperbee": "^2.20.7", "hypercore": "^10.38.1", - "hypercore-byte-stream": "^2.0.1", + "hypercore-byte-stream": "file:../hypercore-byte-stream", "hypercore-crypto": "^3.4.2", "listen-async": "^1.0.0", "streamx": "^2.21.1", diff --git a/test/helpers/index.js b/test/helpers/index.js index a004d37..384d1f0 100644 --- a/test/helpers/index.js +++ b/test/helpers/index.js @@ -33,6 +33,12 @@ function get (link, range = null) { res.on('end', function () { resolve({ status: res.statusCode, data: buf }) }) + res.on('close', function () { + resolve({ status: res.statusCode, data: buf }) + }) + res.on('error', (error) => { + console.error(error) + }) } }) }) From 0dab382b953583ece70a173e2fbb059192857230 Mon Sep 17 00:00:00 2001 From: natzcam Date: Thu, 2 Jan 2025 21:23:36 +0800 Subject: [PATCH 10/18] fix error --- test/blobs.js | 2 +- test/helpers/index.js | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/test/blobs.js b/test/blobs.js index c347400..67fc0f3 100644 --- a/test/blobs.js +++ b/test/blobs.js @@ -67,7 +67,7 @@ test('can get a partial blob from hypercore, but request the whole data', async t.is(res.data, 'Hello World') }) -test.solo('handle out of range header end', async function (t) { +test('handle out of range header end', async function (t) { const store = new Corestore(RAM) const blobs = testHyperblobs(t, store) diff --git a/test/helpers/index.js b/test/helpers/index.js index 384d1f0..004b88f 100644 --- a/test/helpers/index.js +++ b/test/helpers/index.js @@ -36,9 +36,6 @@ function get (link, range = null) { res.on('close', function () { resolve({ status: res.statusCode, data: buf }) }) - res.on('error', (error) => { - console.error(error) - }) } }) }) From 3ef0811844b2395b4934009e6d9912bafb19c126 Mon Sep 17 00:00:00 2001 From: natzcam Date: Thu, 2 Jan 2025 21:24:32 +0800 Subject: [PATCH 11/18] remove local install --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 87ec02d..fd2f35d 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "get-mime-type": "^1.0.2", "hyperbee": "^2.20.7", "hypercore": "^10.38.1", - "hypercore-byte-stream": "file:../hypercore-byte-stream", + "hypercore-byte-stream": "^2.0.1", "hypercore-crypto": "^3.4.2", "listen-async": "^1.0.0", "streamx": "^2.21.1", From 83198ec62c834c197ca22edb6efaa8907773bd5c Mon Sep 17 00:00:00 2001 From: natzcam Date: Thu, 2 Jan 2025 22:16:03 +0800 Subject: [PATCH 12/18] rename vars --- README.md | 35 ++++++++++++++++++++++++++++++++++ test/blobs.js | 44 +++++++++++++++++++++---------------------- test/drives.js | 22 ++++++++++++++++++---- test/helpers/index.js | 13 +++++++------ 4 files changed, 82 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 621d966..c4ef84c 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,41 @@ const link = server.getLink(key, { }) ``` +## API + +#### `const server = new BlobServer(store, options)` + +`store` - Corestore instance + +`options`: +```js +{ + port = 49833, + host = '127.0.0.1', + token = crypto.randomBytes(32), + protocol = 'http', + anyPort = true +} +``` + +#### `await server.listen()` +Listen to http requests. + +#### `const link = server.getLink(key, options)` + +`key` - hypercore or hyperdrive key + +Available `options`: +```js +{ + port = 49833, + host = '127.0.0.1', + token = crypto.randomBytes(32), + protocol = 'http', + anyPort = true +} +``` + ## License Apache-2.0 diff --git a/test/blobs.js b/test/blobs.js index 67fc0f3..305d91d 100644 --- a/test/blobs.js +++ b/test/blobs.js @@ -1,7 +1,7 @@ const test = require('brittle') const RAM = require('random-access-memory') const Corestore = require('corestore') -const { testServeBlobs, request, testHyperblobs } = require('./helpers') +const { testBlobServer, request, testHyperblobs } = require('./helpers') test('can get blob from hypercore', async function (t) { const store = new Corestore(RAM) @@ -10,10 +10,10 @@ test('can get blob from hypercore', async function (t) { const id = await blobs.put(Buffer.from('Hello World')) - const serve = testServeBlobs(t, store) - await serve.listen() + const server = testBlobServer(t, store) + await server.listen() - const res = await request(serve, blobs.core.key, { blob: id }) + const res = await request(server, blobs.core.key, { blob: id }) t.is(res.status, 200) t.is(res.data, 'Hello World') @@ -28,10 +28,10 @@ test('can get blob from hypercore - multiple blocks', async function (t) { const id = await blobs.put(Buffer.from('Hello World')) t.is(id.blockLength, 3) // 3 blocks - const serve = testServeBlobs(t, store) - await serve.listen() + const server = testBlobServer(t, store) + await server.listen() - const res = await request(serve, blobs.core.key, { blob: id }) + const res = await request(server, blobs.core.key, { blob: id }) t.is(res.status, 200) t.is(res.data, 'Hello World') @@ -44,10 +44,10 @@ test('can get a partial blob from hypercore', async function (t) { const id = await blobs.put(Buffer.from('Hello World')) - const serve = testServeBlobs(t, store) - await serve.listen() + const server = testBlobServer(t, store) + await server.listen() - const res = await request(serve, blobs.core.key, { blob: id, range: 'bytes=3-7' }) + const res = await request(server, blobs.core.key, { blob: id, range: 'bytes=3-7' }) t.is(res.status, 206) t.is(res.data, 'lo Wo') }) @@ -59,10 +59,10 @@ test('can get a partial blob from hypercore, but request the whole data', async const id = await blobs.put(Buffer.from('Hello World')) - const serve = testServeBlobs(t, store) - await serve.listen() + const server = testBlobServer(t, store) + await server.listen() - const res = await request(serve, blobs.core.key, { blob: id, range: 'bytes=0-10' }) + const res = await request(server, blobs.core.key, { blob: id, range: 'bytes=0-10' }) t.is(res.status, 206) t.is(res.data, 'Hello World') }) @@ -74,10 +74,10 @@ test('handle out of range header end', async function (t) { const id = await blobs.put(Buffer.from('Hello World')) - const serve = testServeBlobs(t, store) - await serve.listen() + const server = testBlobServer(t, store) + await server.listen() - const res = await request(serve, blobs.core.key, { blob: id, range: 'bytes=0-20' }) + const res = await request(server, blobs.core.key, { blob: id, range: 'bytes=0-20' }) t.is(res.status, 206) t.is(res.data, 'Hello World') }) @@ -89,10 +89,10 @@ test('handle range header without end', async function (t) { const id = await blobs.put(Buffer.from('Hello World')) - const serve = testServeBlobs(t, store) - await serve.listen() + const server = testBlobServer(t, store) + await server.listen() - const res = await request(serve, blobs.core.key, { blob: id, range: 'bytes=2-' }) + const res = await request(server, blobs.core.key, { blob: id, range: 'bytes=2-' }) t.is(res.status, 206) t.is(res.data, 'llo World') }) @@ -104,10 +104,10 @@ test('handle invalid range header', async function (t) { const id = await blobs.put(Buffer.from('Hello World')) - const serve = testServeBlobs(t, store) - await serve.listen() + const server = testBlobServer(t, store) + await server.listen() - const res = await request(serve, blobs.core.key, { blob: id, range: 'testing' }) + const res = await request(server, blobs.core.key, { blob: id, range: 'testing' }) t.is(res.status, 200) t.is(res.data, 'Hello World') }) diff --git a/test/drives.js b/test/drives.js index 594c2b3..3005133 100644 --- a/test/drives.js +++ b/test/drives.js @@ -1,7 +1,7 @@ const test = require('brittle') const RAM = require('random-access-memory') const Corestore = require('corestore') -const { testHyperdrive, testServeBlobs, request } = require('./helpers') +const { testHyperdrive, testBlobServer, request } = require('./helpers') test('can get file from hyperdrive', async function (t) { const store = new Corestore(RAM) @@ -9,10 +9,24 @@ test('can get file from hyperdrive', async function (t) { const drive = testHyperdrive(t, store) await drive.put('/file.txt', 'Here') - const serve = testServeBlobs(t, store) - await serve.listen() + const server = testBlobServer(t, store) + await server.listen() - const res = await request(serve, drive.key, { filename: '/file.txt' }) + const res = await request(server, drive.key, { filename: '/file.txt' }) t.is(res.status, 200) t.is(res.data, 'Here') }) + +test('404 if file not found', async function (t) { + const store = new Corestore(RAM) + + const drive = testHyperdrive(t, store) + await drive.put('/file.txt', 'Here') + + const server = testBlobServer(t, store) + await server.listen() + + const res = await request(server, drive.key, { filename: '/testing.txt' }) + t.is(res.status, 404) + t.is(res.data, '') +}) diff --git a/test/helpers/index.js b/test/helpers/index.js index 004b88f..f00a8ee 100644 --- a/test/helpers/index.js +++ b/test/helpers/index.js @@ -1,11 +1,12 @@ const http = require('http') -const ServeBlobs = require('../../index.js') +const BlobServer = require('../../index.js') const Hyperdrive = require('hyperdrive') const Hyperblobs = require('hyperblobs') module.exports = { request, - testServeBlobs, + get, + testBlobServer, testHyperblobs, testHyperdrive } @@ -47,10 +48,10 @@ async function request (serve, key, opts) { return get(link, opts.range) } -function testServeBlobs (t, store, opts) { - const serve = new ServeBlobs(store, opts) - t.teardown(() => serve.close()) - return serve +function testBlobServer (t, store, opts) { + const server = new BlobServer(store, opts) + t.teardown(() => server.close()) + return server } function testHyperblobs (t, store) { From fa76298daa54305fe13d02b4c62d0abf87a42f46 Mon Sep 17 00:00:00 2001 From: natzcam Date: Thu, 2 Jan 2025 22:28:43 +0800 Subject: [PATCH 13/18] more tests --- README.md | 16 ++++++++++------ test/drives.js | 18 +++++++++++++++++- test/helpers/index.js | 4 ++-- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index c4ef84c..ad0b887 100644 --- a/README.md +++ b/README.md @@ -54,14 +54,18 @@ Listen to http requests. `key` - hypercore or hyperdrive key -Available `options`: +`options`: ```js { - port = 49833, - host = '127.0.0.1', - token = crypto.randomBytes(32), - protocol = 'http', - anyPort = true + host = this.host, + port = this.port, + protocol = this.protocol, + filename = null, + blob = null, + url = true, + mimetype = filename ? getMimeType(filename) : 'application/octet-stream', + mimeType = mimetype, + type = mimeType } ``` diff --git a/test/drives.js b/test/drives.js index 3005133..f66b5b3 100644 --- a/test/drives.js +++ b/test/drives.js @@ -1,7 +1,7 @@ const test = require('brittle') const RAM = require('random-access-memory') const Corestore = require('corestore') -const { testHyperdrive, testBlobServer, request } = require('./helpers') +const { testHyperdrive, testBlobServer, request, get } = require('./helpers') test('can get file from hyperdrive', async function (t) { const store = new Corestore(RAM) @@ -30,3 +30,19 @@ test('404 if file not found', async function (t) { t.is(res.status, 404) t.is(res.data, '') }) + +test('404 if token is invalid', async function (t) { + const store = new Corestore(RAM) + + const drive = testHyperdrive(t, store) + await drive.put('/file.txt', 'Here') + + const server = testBlobServer(t, store) + await server.listen() + + const link = server.getLink(drive.key, { filename: '/testing.txt' }) + const res = await get(link.replace('token=', 'token=breakme')) + + t.is(res.status, 404) + t.is(res.data, '') +}) diff --git a/test/helpers/index.js b/test/helpers/index.js index f00a8ee..7ef0e25 100644 --- a/test/helpers/index.js +++ b/test/helpers/index.js @@ -42,8 +42,8 @@ function get (link, range = null) { }) } -async function request (serve, key, opts) { - const link = serve.getLink(key, opts) +async function request (server, key, opts) { + const link = server.getLink(key, opts) return get(link, opts.range) } From 7cfb6daf7177a5b4e03c6e7dbb2df01d69331190 Mon Sep 17 00:00:00 2001 From: natzcam Date: Fri, 3 Jan 2025 00:57:17 +0800 Subject: [PATCH 14/18] fix bug on suspend/resume --- README.md | 26 +++++++++++--------------- index.js | 3 ++- test/drives.js | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index ad0b887..278511c 100644 --- a/README.md +++ b/README.md @@ -39,35 +39,31 @@ const link = server.getLink(key, { `options`: ```js { - port = 49833, - host = '127.0.0.1', - token = crypto.randomBytes(32), - protocol = 'http', - anyPort = true + port // defaults to 49833, + host // defaults to '127.0.0.1', + token // server token + protocol // 'http' | 'https' } ``` #### `await server.listen()` -Listen to http requests. +Listen to requests #### `const link = server.getLink(key, options)` +Generates the url used to fetch data + `key` - hypercore or hyperdrive key `options`: ```js { - host = this.host, - port = this.port, - protocol = this.protocol, - filename = null, - blob = null, - url = true, - mimetype = filename ? getMimeType(filename) : 'application/octet-stream', - mimeType = mimetype, - type = mimeType + filename | blob } ``` +`filename` - hyperdrive filename + +`blob` - blob ID in the form of `{ blockOffset, blockLength, byteOffset, byteLength}` ## License diff --git a/index.js b/index.js index 35e868d..f69418f 100644 --- a/index.js +++ b/index.js @@ -60,7 +60,7 @@ module.exports = class ServeBlobs { _onconnection (socket) { if (this.suspending) { - this.connection.destroy() + socket.destroy() return } @@ -193,6 +193,7 @@ module.exports = class ServeBlobs { async _resume () { if (this.suspending) await this.suspending + this.suspending = null if (this.server !== null) { await this._closeAll(true) this.server.ref() diff --git a/test/drives.js b/test/drives.js index f66b5b3..62557a1 100644 --- a/test/drives.js +++ b/test/drives.js @@ -46,3 +46,39 @@ test('404 if token is invalid', async function (t) { t.is(res.status, 404) t.is(res.data, '') }) + +test('sending request while suspended', async function (t) { + const store = new Corestore(RAM) + + const drive = testHyperdrive(t, store) + await drive.put('/file.txt', 'Here') + + const server = testBlobServer(t, store) + await server.listen() + + await server.suspend() + + try { + await request(server, drive.key, { filename: '/file.txt' }) + t.fail('request should fail') + } catch (err) { + t.is(err.message, 'read ECONNRESET') + } +}) + +test.solo('sending request after resume', async function (t) { + const store = new Corestore(RAM) + + const drive = testHyperdrive(t, store) + await drive.put('/file.txt', 'Here') + + const server = testBlobServer(t, store) + await server.listen() + + await server.suspend() + await server.resume() + + const res = await request(server, drive.key, { filename: '/file.txt' }) + t.is(res.status, 200) + t.is(res.data, 'Here') +}) From f1d618f2aa5d678f3705a01c47eb54fcbe4f7bea Mon Sep 17 00:00:00 2001 From: natzcam Date: Fri, 3 Jan 2025 01:02:06 +0800 Subject: [PATCH 15/18] resume readme --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 278511c..f43abde 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,9 @@ Generates the url used to fetch data `options`: ```js { + host // custom host + port // custom port + protocol: 'http' | 'https', filename | blob } ``` @@ -65,6 +68,14 @@ Generates the url used to fetch data `blob` - blob ID in the form of `{ blockOffset, blockLength, byteOffset, byteLength}` +#### `await server.suspend()` + +Let the instance know you wanna suspend so it can make relevant changes. + +#### `await server.resume()` + +Let the instance know you wanna resume from suspension. Will rebind the server etc. + ## License Apache-2.0 From 15e35415fe7297db6c1d6001c81f057b29574a09 Mon Sep 17 00:00:00 2001 From: natzcam Date: Fri, 3 Jan 2025 01:05:11 +0800 Subject: [PATCH 16/18] remove solo --- test/drives.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/drives.js b/test/drives.js index 62557a1..199ba0c 100644 --- a/test/drives.js +++ b/test/drives.js @@ -66,7 +66,7 @@ test('sending request while suspended', async function (t) { } }) -test.solo('sending request after resume', async function (t) { +test('sending request after resume', async function (t) { const store = new Corestore(RAM) const drive = testHyperdrive(t, store) From 0dbd5c754eb2352b90c1af1df3049b6f2ef7f2d7 Mon Sep 17 00:00:00 2001 From: natzcam Date: Fri, 3 Jan 2025 01:10:28 +0800 Subject: [PATCH 17/18] accept generic error --- test/drives.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/drives.js b/test/drives.js index 199ba0c..f329044 100644 --- a/test/drives.js +++ b/test/drives.js @@ -62,7 +62,7 @@ test('sending request while suspended', async function (t) { await request(server, drive.key, { filename: '/file.txt' }) t.fail('request should fail') } catch (err) { - t.is(err.message, 'read ECONNRESET') + t.ok(err) } }) From 73c7e7b5a2bd126cfee19f233e0ad6fab54c9f0a Mon Sep 17 00:00:00 2001 From: natzcam Date: Fri, 3 Jan 2025 01:21:26 +0800 Subject: [PATCH 18/18] add bit about Range --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index f43abde..dd2a4d1 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,14 @@ Generates the url used to fetch data `blob` - blob ID in the form of `{ blockOffset, blockLength, byteOffset, byteLength}` +When downloading blobs, you can set the `Range` header to download sections of data, implement pause/resume download functionality. Offsets are zero-indexed & inclusive + +``` +Range: bytes=- +Range: bytes=0-300 +Range: bytes=2- +``` + #### `await server.suspend()` Let the instance know you wanna suspend so it can make relevant changes.