Local Datasets WIP

.eslintignore

@@ -0,0 +1,2 @@
+**/*.test.js
+**/tests/*.test.js

README.md

@@ -57,6 +57,12 @@ Access-Control-Allow-Methods: GET, POST, PUT, DELETE, OPTIONS
 Access-Control-Allow-Headers: Content-Type, Authorization
 ```
 
+## Local Datasets
+
+Tractoscope has support for local datasets via a locally hosted server
+The code for this server is found in the "local-server" folder. For
+more information view the README in that folder.
+
 ## Project setup
 ```
 npm install

local-server/Dockerfile

@@ -0,0 +1,16 @@
+FROM python:3.8-alpine
+
+WORKDIR /app
+
+COPY . /app
+
+# install any dependencies, currently none exist
+RUN if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
+EXPOSE 8000
+
+# make a datasets directory
+RUN mkdir -p /datasets
+
+#run the python server on port 8000
+CMD ["python", "host.py","--port","8000"]

local-server/README

@@ -0,0 +1,47 @@
+# Tractoscope Local Server
+
+This python script will create a local server that serves GET request
+and a special request type "listFiles" which returns a json containing
+the files in a given folder.
+
+## Usage
+
+To run the project, use the following command, optionally specifing the port:
+
+```bash
+python host.py --port 8000
+```
+
+## You can also use the docker image to run the server
+
+To give the image access to you dataset run the docker container
+with the -v argument, linking a folder named datasets, containing your
+datasets on your local machine to one inside the container.
+
+replace 'externalPort' with whatever port you would
+like to bind the server to
+
+```bash
+docker run -v /path/to/datasets:/app/datasets -p 'externalPort':8000 tractoscope-local-server
+```
+
+## To configure with tractoscope
+
+Once you have setup the server you need to add the datasets in tractoscopes datasets.json file, which can be found in the public folder. Set the bucket to "localhost:{port}", replacing {port} with whatever port the server is bound to. Asuming all of your datasets are stored correctly inside of the datasets folder, your prefix should be set to "/datasets/yourDataset".
+
+heres what an example config might look like.
+
+> Note: you do not need the "participantsSize" parameter due to the fact that the local server has no limit on the length of its listFiles response.
+
+
+```json
+{
+    "yourDataset": {
+        "bucket": "localhost:8000",
+        "prefix": "/datasets/yourDataset",
+        "scans": [ "someScan" ],
+        "trkFiles": [ "someTrk" ],
+    }
+}
+```
+

local-server/host.py

@@ -0,0 +1,30 @@
+import socketserver
+import socket
+import argparse
+from requestHandler import requestHandler
+
+# Parse command-line arguments
+parser = argparse.ArgumentParser()
+parser.add_argument("--port", type=int, default=-1, help="the port number to start at")
+args = parser.parse_args()
+port = None
+
+if args.port == -1:
+    print("no port provided, finding open port")
+    port = 8000
+    while True:
+        try:
+            with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+                s.bind(("localhost", port))
+            break
+        except OSError:
+            port += 1
+else:
+    port = args.port
+
+if __name__ == "__main__":
+    # Set up the server on the specified port
+    with socketserver.TCPServer(("", port), requestHandler) as httpd:
+        print(f"Serving on port {port}")
+        # Start the server
+        httpd.serve_forever()

local-server/requestHandler.py

@@ -0,0 +1,64 @@
+import http.server
+import socketserver
+import json
+import xml.etree.ElementTree as ET
+from urllib.parse import urlparse, parse_qs
+import os
+
+class requestHandler(http.server.SimpleHTTPRequestHandler):
+    def do_GET(self):
+        parsed_url = urlparse(self.path)
+        path = parsed_url.path
+        query = parsed_url.query
+        if path == '/listFiles':
+            params = parse_qs(query)
+            file_path = params.get('path')
+            recursive = params.get('recursive')
+
+            if recursive:
+                recursive = recursive[0]
+                if recursive == 'true':
+                    recursive = True
+                else:
+                    recursive = False
+            else:
+                recursive = False
+
+            # Check if path was provided
+            if not file_path:
+                self.send_response(400)
+                self.send_header('Content-type', 'application/json')
+                self.end_headers()
+                response = {"error": "No path provided"}
+                self.wfile.write(json.dumps(response).encode())
+
+            else:
+                file_path = file_path[0]
+
+                #get all file names in the directory
+                files = self.listFiles(file_path, recursive)
+                response = {}
+                response['Contents'] = files
+
+                self.send_response(200)
+                self.send_header('Content-type', 'application/json')
+                self.end_headers()
+                self.wfile.write(json.dumps(response).encode())
+
+        else:
+            super().do_GET()
+
+    def listFiles(self, path, recursive):
+        output = []
+        # get all file names in the directory
+        try:
+            files = os.listdir(path)
+        except FileNotFoundError:
+            return
+        # if recursive is true, then get all files in subdirectories
+        for file in files:
+            output.append({'name': file, 'Key': os.path.join(path, file)})
+            new_path = os.path.join(path, file)
+            if recursive and os.path.isdir(new_path):
+                output.extend(self.listFiles(new_path, recursive))
+        return output

local-server/test.py

@@ -0,0 +1,69 @@
+import unittest
+import socketserver
+import socket
+import http.client
+import argparse
+import threading
+import json
+from requestHandler import requestHandler
+
+class TestRequestHandler(unittest.TestCase):
+    def setUp(self):
+        self.port = 8000
+        # find port
+        while True:
+            try:
+                with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+                    s.bind(("localhost", self.port))
+                break
+            except OSError:
+                self.port += 1
+        # setup server on thread
+        self.server_thread = threading.Thread(target=self.startServer)
+        self.server_thread.start()
+
+    def startServer(self):
+        with socketserver.TCPServer(("", self.port), requestHandler) as httpd:
+            print(f"Serving on port {self.port}")
+            # Start the server
+            httpd.serve_forever()
+
+    def test_do_listFiles(self):
+
+        conn = http.client.HTTPConnection('localhost', self.port)
+        conn.request('GET', '/listFiles?path=testFolder/&recursive=true')
+
+        response = conn.getresponse()
+
+        # check response status
+        self.assertEqual(response.status, 200)
+
+        #read json
+        response_contents = response.read()
+        response_dict = json.loads(response_contents)
+
+        # check that response contains Contents and IsTruncated
+        self.assertTrue('Contents' in response_dict)
+        self.assertTrue('IsTruncated' in response_dict)
+
+        #check that contents is a list
+        self.assertTrue(isinstance(response_dict['Contents'], list))
+
+        #check that contents contains the correct files
+        contents = response_dict['Contents']
+        self.assertTrue(all("name" in item and "Key" in item for item in contents))
+        #check that testFile was found
+        self.assertTrue(any(item['name'] == 'testFile.txt' for item in contents))
+        #check that recursiveTestFile was found
+        self.assertTrue(any(item['name'] == 'recursiveTestFile.txt' for item in contents))
+
+    def test_do_listFiles_no_path(self):
+        conn = http.client.HTTPConnection('localhost', self.port)
+        conn.request('GET', '/listFiles')
+        response = conn.getresponse()
+        #check response status
+        self.assertEqual(response.status, 400)
+
+
+if __name__ == '__main__':
+    unittest.main()

local-server/testFolder/recurisiveTestFolder/recursiveTestFile.txt

@@ -0,0 +1 @@
+this is also a test file for unit testing

local-server/testFolder/testFile.txt

@@ -0,0 +1 @@
+this is a test file for unit testing.