v1.0.0 (#86)

* Improve sync restart

* Resolve cursor issue

* Fix indexed tag query

* Fix sub query whereIn

* Resolve [POST] /tx issue

* Manifest Middleware

* Manifest Table SQL

* Manifest, faster redirect

* Update manifest prefix

* Updated documentation

* Added CORS middleware

Author: TheLoneRonin

.env.dev

@@ -6,8 +6,6 @@ DATABASE_PASSWORD=arweave
 DATABASE_NAME=arweave
 
 ENVIRONMENT=public
-MANIFESTS=0
-BIP39=0
 PORT=3000
 
 PARALLEL=4
@@ -21,4 +19,4 @@ CACHING=1
 CACHE_FOLDER=/gateway/cache
 CACHE_OFFSET=0
 
-MANIFEST_PREFIX=http://localhost:3000
+MANIFEST_PREFIX=amp-gw.online

.env.docker

@@ -6,8 +6,6 @@ DATABASE_PASSWORD=arweave
 DATABASE_NAME=arweave
 
 ENVIRONMENT=public
-MANIFESTS=0
-BIP39=0
 PORT=3000
 
 PARALLEL=4
@@ -21,4 +19,4 @@ CACHING=1
 CACHE_FOLDER=/gateway/cache
 CACHE_OFFSET=0
 
-MANIFEST_PREFIX=https://gateway.amplify.host
+MANIFEST_PREFIX=amp-gw.online

README.md

@@ -36,8 +36,6 @@ DATABASE_PASSWORD=arweave
 DATABASE_NAME=arweave
 
 ENVIRONMENT=public
-MANIFESTS=0
-BIP39=0
 PORT=3000
 
 PARALLEL=4
@@ -48,7 +46,7 @@ CACHING=1
 CACHE_FOLDER=/gateway/cache
 CACHE_OFFSET=0
 
-MANIFEST_PREFIX=https://gateway.amplify.host
+MANIFEST_PREFIX=amp-gw.online
 ```
 
 Make sure you copy this configuration to `.env`.

bin/index.create.sh

@@ -34,6 +34,8 @@ CREATE INDEX "index_domain_transactions" ON transactions USING BTREE ("domain");
 CREATE INDEX "index_app_transactions" ON transactions USING BTREE ("app");
 --- Transaction App-Name Index
 CREATE INDEX "index_App-Name_transactions" ON transactions USING BTREE ("App-Name");
+--- Transactions created_at index
+CREATE INDEX "transactions_created_at" ON transactions USING BTREE ("created_at");
 
 --- Tag Indices
 --- Tag Transaction Id Index

bin/manifest.table.sql

@@ -0,0 +1,8 @@
+CREATE TABLE manifest (
+    "manifest_url" varchar(64),
+    "manifest_id" varchar(64),
+    "tx_id" varchar(64),
+    "path" text,
+
+    PRIMARY KEY("manifest_id", "tx_id")
+);

docs/DEV.md

@@ -53,8 +53,6 @@ DATABASE_PASSWORD=arweave
 DATABASE_NAME=arweave
 
 ENVIRONMENT=public
-MANIFESTS=0
-BIP39=0
 PORT=3000
 
 PARALLEL=4
@@ -68,7 +66,7 @@ CACHING=1
 CACHE_FOLDER=/gateway/cache
 CACHE_OFFSET=0
 
-MANIFEST_PREFIX=https://gateway.amplify.host
+MANIFEST_PREFIX=amp-gw.online
 ```
 
 Make sure you copy this configuration to `.env`.

docs/MANIFEST.md

@@ -1,52 +1,24 @@
 # Transaction Manifests
 
-### Enabling Transaction Manifests
-
-Transaction Manifests are important to prevent `CORS`. By default, manifests are disabled in the `.env` files. You can enable it by changing:
-
-```bash
-MANIFESTS=0
-```
-
-to
-
-```bash
-MANIFESTS=1
-```
+Transaction manifests are only usable with a valid domain. Please make sure to have a valid domain with wildcard SSL enabled. You can use Let's Encrypt to configure a domain accordingly.
 
 In order to effectively use manifests. You need to have domain wildcards pointing to your domain.
 
-### Using BIP39 or SHA256
+### Configuring Manifests
 
-There are two different types of transaction manifests subdomains. SHA256 and BIP39. You can enable SHA256 subdomains by changing your `.env` file to:
+In order to configure manifests, you need to change the `MANIFEST_PREFIX` environment variable. It should just be your domain name. Simply change it from:
 
-```bash
-BIP39=0
+```conf
+MANIFEST_PREFIX=amp-gw.online
 ```
 
-You can enable BIP39 subdomains by changing the `.env` file to:
+to
 
-```bash
-BIP39=1
+```conf
+MANIFEST_PREFIX=my-actual.domain
 ```
 
-## Manifests for Websites
-
-Manifests are fairly simple for actual domains. Let's say own the domain `amplify.testing123`. Follow these steps to enable transaction manifests on your website.
-
-1. Point your `@` record to your Gateway server.
-
-    - This would be an `A` record for an IP.
-
-    - This would be a `CNAME` record for a service's subdomain (ie: AWS EC2, Heroku)
-
-2. Point your `*.amplify.testing123` record to your Gateway server. Follow the same rules as 1.
-
-3. Test your Gateway endpoints
-    
-    - Run `ping amplify.testing123`. Ensure it runs a valid response.
-
-    - Go to any transaction id on `amplify.testing123/[tx id]` and ensure it redirects to a valid transaction manifest path.
+Once configured and the Gateway points to the same domain. It should be good to go.
 
 ## Manifests for Development
 

docs/README.md

@@ -51,7 +51,7 @@ CACHING=1
 CACHE_FOLDER=/gateway/cache
 CACHE_OFFSET=0
 
-MANIFEST_PREFIX=https://gateway.amplify.host
+MANIFEST_PREFIX=amp-gw.online
 ```
 
 Make sure you copy this configuration to `.env`.

migrations/20200404025828_initialize.ts

@@ -53,6 +53,14 @@ export async function up(knex: Knex) {
         table.timestamp('created_at').defaultTo(knex.fn.now());
 
         table.primary(['tx_id', 'index'], 'pkey_tags');
+      })
+      .createTable('manifest', (table) => {
+        table.string('manifest_url', 64).notNullable();
+        table.string('manifest_id', 64).notNullable();
+        table.string('tx_id', 64).notNullable();
+        table.text('path').notNullable();
+
+        table.primary(['manifest_url', 'tx_id'], 'pkey_manifest');
       });
 }
 
@@ -61,5 +69,6 @@ export async function down(knex: Knex) {
       .withSchema(process.env.ENVIRONMENT || 'public')
       .dropTableIfExists('transactions')
       .dropTableIfExists('blocks')
-      .dropTableIfExists('tags');
+      .dropTableIfExists('tags')
+      .dropTableIfExists('manifest');
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arweave/gateway",
-  "version": "0.11.3",
+  "version": "1.0.0",
   "main": "dist/src/Gateway.js",
   "repository": "[email protected]:ArweaveTeam/gateway.git",
   "author": "Arweave <[email protected]>",
@@ -34,6 +34,7 @@
     "bip39": "^3.0.3",
     "body-parser": "^1.19.0",
     "colors": "^1.4.0",
+    "cors": "^2.8.5",
     "crypto-random-string": "^3.3.1",
     "detect-mocha": "^0.1.0",
     "dotenv": "^8.2.0",
@@ -47,8 +48,9 @@
     "graphql-fields": "^2.0.3",
     "js-sha256": "^0.9.0",
     "knex": "^0.21.17",
-    "koi-logs": "https://github.com/open-koi/logs",
+    "koi-logs": "https://github.com/koii-network/logs",
     "lodash": "^4.17.20",
+    "mime-types": "^2.1.31",
     "moment": "^2.29.1",
     "morgan": "^1.10.0",
     "node-cron": "^2.0.3",
@@ -67,6 +69,7 @@
     "@types/express": "^4.17.11",
     "@types/graphql-fields": "^1.3.3",
     "@types/lodash": "^4.14.168",
+    "@types/mime-types": "^2.1.0",
     "@types/mocha": "^8.2.1",
     "@types/morgan": "^1.9.2",
     "@types/node": "^14.14.22",

src/Gateway.ts

@@ -1,13 +1,15 @@
 import 'colors';
 import express, {Express} from 'express';
 import {config} from 'dotenv';
-import {corsMiddleware} from './middleware/cors.middleware';
+import cors from 'cors';
 import {jsonMiddleware} from './middleware/json.middleware';
 import {logMiddleware} from './middleware/log.middleware';
+import {manifestMiddleware} from './middleware/manifest.middleware';
 import {log} from './utility/log.utility';
 import {sessionMiddleware, sessionPinningMiddleware} from './utility/session.utility';
 import {graphServer} from './graphql/server.graphql';
 import {statusRoute} from './route/status.route';
+import {transactionRoute} from './route/transaction.route';
 import {syncRoute} from './route/sync.route';
 import {proxyRoute} from './route/proxy.route';
 import {dataRouteRegex, dataRoute} from './route/data.route';
@@ -22,12 +24,13 @@ export const app: Express = express();
 export function start() {
   app.set('trust proxy', 1);
 
-  app.use(corsMiddleware);
+  app.use(cors());
   app.use(jsonMiddleware);
   app.use(logMiddleware);
   app.use(sessionMiddleware);
   app.use(sessionPinningMiddleware);
   app.use(koiLogger.logger);
+  app.use(manifestMiddleware);
 
   app.get('/', statusRoute);
   app.get('/status', syncRoute);
@@ -36,6 +39,7 @@ export function start() {
 
   graphServer({introspection: true, playground: true}).applyMiddleware({app, path: '/graphql'});
 
+  app.post('/tx', transactionRoute);
   app.get('/peers', peerRoute);
   app.get('/logs', koiLogsRoute);
   app.get('/logs/raw', koiLogsRawRoute);

src/database/sync.database.ts

@@ -77,7 +77,7 @@ export async function startSync() {
 
 export async function parallelize(height: number) {
   clearTimeout(timer);
-  timer = setTimeout(() => {
+  timer = setTimeout(async () => {
     log.info('[database] sync timed out, restarting server');
     process.exit();
   }, 300 * 1000);
@@ -159,7 +159,7 @@ export async function storeBlock(height: number, retry: number = 0) {
     if (SIGKILL === false) {
       if (retry >= 25) {
         log.info(`[snapshot] there were problems retrieving ${height}, restarting the server`);
-        process.exit();
+        await startSync();
       } else {
         log.info(`[snapshot] could not retrieve block at height ${height}, retrying`);
         await storeBlock(height, retry + 1);

src/graphql/query.graphql.ts

@@ -1,7 +1,7 @@
 import {config} from 'dotenv';
 import {QueryBuilder} from 'knex';
-import {indices} from '../utility/order.utility';
 import {connection} from '../database/connection.database';
+import {indices} from '../utility/order.utility';
 import {ISO8601DateTimeString} from '../utility/encoding.utility';
 import {TagFilter} from './types';
 import {tagToB64, toB64url} from '../query/transaction.query';
@@ -30,7 +30,7 @@ export interface QueryParams {
   offset?: number;
   select?: any;
   blocks?: boolean;
-  since?: ISO8601DateTimeString;
+  since?: ISO8601DateTimeString | string;
   sortOrder?: TxSortOrder;
   status?: 'any' | 'confirmed' | 'pending';
   pendingMinutes?: number;
@@ -41,7 +41,7 @@ export interface QueryParams {
 export async function generateQuery(params: QueryParams): Promise<QueryBuilder> {
   const {to, from, tags, id, ids, status = 'confirmed', select} = params;
   const {limit = 10, sortOrder = 'HEIGHT_DESC'} = params;
-  const {since, offset = 0, minHeight = -1, maxHeight = -1} = params;
+  const {since = new Date().toISOString(), offset = 0, minHeight = -1, maxHeight = -1} = params;
 
   const query = connection
       .queryBuilder()
@@ -76,55 +76,29 @@ export async function generateQuery(params: QueryParams): Promise<QueryBuilder>
 
   if (tags) {
     const tagsConverted = tagToB64(tags);
-    const names: Array<string> = [];
-    const values: Array<string> = [];
-
-    const subQuery = connection
-        .queryBuilder()
-        .select('*')
-        .from('tags');
 
-    let runSubQuery = false;
-
-    for (let i = 0; i < tagsConverted.length; i++) {
-      const tag = tagsConverted[i];
+    tagsConverted.forEach((tag) => {
       let indexed = false;
 
-      for (let ii = 0; ii < indices.length; ii++) {
-        const index = toB64url(indices[ii]);
+      for (let i = 0; i < indices.length; i++) {
+        const index = toB64url(indices[i]);
 
         if (tag.name === index) {
+          query.whereIn(`transactions.${indices[i]}`, tag.values);
           indexed = true;
-          query.whereIn(`transactions.${index}`, tag.values);
         }
       }
 
       if (indexed === false) {
-        names.push(tag.name);
-        values.push.apply(values, tag.values);
-
-        runSubQuery = true;
+        query.whereIn('transactions.id', (subQuery) => {
+          return subQuery
+              .select('tx_id')
+              .from('tags')
+              .where('tags.name', tag.name)
+              .whereIn('tags.value', tag.values);
+        });
       }
-    }
-
-    subQuery.whereIn('name', names);
-    subQuery.whereIn('value', values);
-
-    if (runSubQuery) {
-      const results = await subQuery
-          .limit(limit)
-          .offset(offset)
-          .orderByRaw(tagOrderByClauses[sortOrder]);
-
-      const tx_ids = [];
-
-      for (let i = 0; i < results.length; i++) {
-        const {tx_id} = results[i];
-        tx_ids.push(tx_id);
-      }
-
-      query.whereIn('transactions.id', tx_ids);
-    }
+    });
   }
 
   if (minHeight >= 0) {
@@ -135,13 +109,13 @@ export async function generateQuery(params: QueryParams): Promise<QueryBuilder>
     query.where('transactions.height', '<=', maxHeight);
   }
 
-  query.limit(limit).offset(offset);
-
   if (Object.keys(orderByClauses).includes(sortOrder)) {
     query.orderByRaw(orderByClauses[sortOrder]);
   }
 
-  query.orderByRaw('transactions.created_at DESC');
+  query.limit(limit).offset(offset);
+
+  query.orderByRaw('transactions.id ASC');
 
   return query;
 }