Draft: Experiment on dataset download

.gitmodules

@@ -1,6 +1,9 @@
 [submodule "public/libs/converter"]
 	path = src/libs/converter
-	url = https://github.com/yaptide/converter
+	url = ../converter
 	branch = master
 	shallow = true
 
+[submodule "src/libs/geant4_web/geant-web-stubs"]
+	path = src/libs/geant4_web/geant-web-stubs
+	url = ../geant-web-stubs

config-overrides.js

@@ -1,3 +1,5 @@
+const webpack = require('webpack');
+
 module.exports = function override(webpackConfig) {
 	// react-dnd
 	webpackConfig.module.rules.unshift({
@@ -7,6 +9,12 @@ module.exports = function override(webpackConfig) {
 		}
 	});
 
+	webpackConfig.plugins.push(
+		new webpack.IgnorePlugin({
+			resourceRegExp: /geant4_wasm\.wasm$/
+		})
+	);
+
 	// react-dnd
 	webpackConfig.resolve.alias = {
 		...webpackConfig.resolve.alias,

src/WrapperApp/WrapperApp.tsx

@@ -3,6 +3,7 @@ import { styled } from '@mui/material/styles';
 import { SyntheticEvent, useEffect, useState } from 'react';
 
 import { useConfig } from '../config/ConfigService';
+import { useDatasetDownloadManager } from '../libs/geant4_web/DatasetDownloadManager';
 import { useAuth } from '../services/AuthService';
 import { FullSimulationData } from '../services/ShSimulatorService';
 import { useStore } from '../services/StoreService';
@@ -50,6 +51,12 @@ function WrapperApp() {
 	const [providedInputFiles, setProvidedInputFiles] = useState<SimulationInputFiles>();
 	const [highlightRunForm, setHighLightRunForm] = useState(false);
 
+	const {
+		managerState: geant4DownloadManagerState,
+		datasetStates: geant4DatasetStates,
+		startDownload: geant4DatasetDownload
+	} = useDatasetDownloadManager();
+
 	useEffect(() => {
 		if (Object.keys(providedInputFiles ?? {}).length > 0) {
 			setHighLightRunForm(true);
@@ -206,6 +213,9 @@ function WrapperApp() {
 						highlight={highlightRunForm}
 						clearInputFiles={() => setProvidedInputFiles(undefined)}
 						runSimulation={runSimulation}
+						geant4DownloadManagerState={geant4DownloadManagerState}
+						geant4DatasetDownloadStart={geant4DatasetDownload}
+						geant4DatasetStates={geant4DatasetStates}
 					/>
 				</TabPanel>
 				{/* end Simulations screen */}

src/WrapperApp/components/Simulation/Geant4Datasets.tsx

@@ -0,0 +1,103 @@
+import CheckIcon from '@mui/icons-material/Check';
+import ExpandMoreIcon from '@mui/icons-material/ExpandMore';
+import {
+	AccordionDetails,
+	AccordionSummary,
+	Box,
+	Button,
+	LinearProgress,
+	Typography,
+	useTheme
+} from '@mui/material';
+import { useState } from 'react';
+
+import {
+	DatasetDownloadStatus,
+	DatasetStatus,
+	DownloadManagerStatus
+} from '../../../libs/geant4_web/DatasetDownloadManager';
+import StyledAccordion from '../../../shared/components/StyledAccordion';
+
+export interface Geant4DatasetsProps {
+	geant4DownloadManagerState: DownloadManagerStatus;
+	geant4DatasetStates: DatasetStatus[];
+	geant4DatasetDownloadStart: () => void;
+}
+
+function DatasetCurrentStatus(props: { status: DatasetStatus }) {
+	const { status } = props;
+
+	return (
+		<Box sx={{ pb: 1 }}>
+			<Box sx={{ display: 'flex', alignItems: 'center' }}>
+				<Typography>{status.name}</Typography>
+				{status.status === DatasetDownloadStatus.DONE && (
+					<CheckIcon
+						color='primary'
+						sx={{ marginLeft: 1 }}
+					/>
+				)}
+			</Box>
+			{(status.status === DatasetDownloadStatus.DOWNLOADING ||
+				status.status === DatasetDownloadStatus.PROCESSING) && (
+				<Box sx={{ display: 'flex', alignItems: 'center' }}>
+					<LinearProgress
+						variant='determinate'
+						value={(status.done! / status.total!) * 100}
+						sx={{ flexGrow: 1, marginRight: 1 }}
+					/>
+					<Typography sx={{ width: 48 }}>
+						{Math.round((status.done! / status.total!) * 10000) / 100}%
+					</Typography>
+				</Box>
+			)}
+			{status.status === DatasetDownloadStatus.IDLE && <LinearProgress />}
+		</Box>
+	);
+}
+
+export function Geant4Datasets(props: Geant4DatasetsProps) {
+	const theme = useTheme();
+	const { geant4DownloadManagerState, geant4DatasetStates, geant4DatasetDownloadStart } = props;
+	const [open, setOpen] = useState(true);
+
+	return (
+		<StyledAccordion
+			expanded={open}
+			sx={{
+				margin: `0 ${theme.spacing(1)} ${theme.spacing(1)} ${theme.spacing(1)}`
+			}}>
+			<AccordionSummary
+				expandIcon={<ExpandMoreIcon />}
+				onClick={() => setOpen(!open)}>
+				<Typography
+					textTransform='none'
+					fontSize={16}>
+					Datasets download
+				</Typography>
+			</AccordionSummary>
+			<AccordionDetails
+				sx={{
+					display: 'flex',
+					flexDirection: 'column'
+				}}>
+				{geant4DownloadManagerState === DownloadManagerStatus.IDLE && (
+					<Button
+						onClick={geant4DatasetDownloadStart}
+						variant='contained'>
+						Start download
+					</Button>
+				)}
+				{geant4DatasetStates.map(status => (
+					<DatasetCurrentStatus
+						status={status}
+						key={status.name}
+					/>
+				))}
+				{geant4DownloadManagerState === DownloadManagerStatus.ERROR && (
+					<Typography>Something went wrong</Typography>
+				)}
+			</AccordionDetails>
+		</StyledAccordion>
+	);
+}

src/WrapperApp/components/Simulation/RunSimulationPanel.tsx

@@ -3,14 +3,21 @@ import CloudOff from '@mui/icons-material/CloudOff';
 import { AccordionDetails, AccordionSummary, Link, Typography, useTheme } from '@mui/material';
 
 import { useConfig } from '../../../config/ConfigService';
+import {
+	DatasetStatus,
+	DownloadManagerStatus
+} from '../../../libs/geant4_web/DatasetDownloadManager';
 import { useAuth } from '../../../services/AuthService';
 import { useStore } from '../../../services/StoreService';
 import StyledAccordion from '../../../shared/components/StyledAccordion';
 import { SimulatorNames, SimulatorType } from '../../../types/RequestTypes';
+import { Geant4Datasets, Geant4DatasetsProps } from './Geant4Datasets';
 import RecentSimulations from './RecentSimulations';
 import { RunSimulationForm, RunSimulationFormProps } from './RunSimulationForm';
 
-export default function RunSimulationPanel(props: RunSimulationFormProps) {
+export type RunSimulationPanelProps = RunSimulationFormProps & Geant4DatasetsProps;
+
+export default function RunSimulationPanel(props: RunSimulationPanelProps) {
 	const theme = useTheme();
 	const { demoMode } = useConfig();
 	const { yaptideEditor } = useStore();
@@ -23,6 +30,9 @@ export default function RunSimulationPanel(props: RunSimulationFormProps) {
 	return showRunForm ? (
 		<>
 			<RunSimulationForm {...props} />
+			{yaptideEditor?.contextManager.currentSimulator === SimulatorType.GEANT4 && (
+				<Geant4Datasets {...props} />
+			)}
 			<RecentSimulations />
 		</>
 	) : (

src/libs/geant4_web/DatasetDownloadManager.ts

@@ -0,0 +1,124 @@
+import { useCallback, useEffect, useState } from 'react';
+
+export enum DownloadManagerStatus {
+	IDLE,
+	WORKING,
+	FINISHED,
+	ERROR,
+}
+
+export enum DatasetDownloadStatus {
+	IDLE,
+	DOWNLOADING,
+	PROCESSING,
+	DONE,
+}
+
+export interface DatasetStatus {
+	name: string,
+	status: DatasetDownloadStatus,
+	done?: number,
+	total?: number,
+}
+
+const idleRegex = /IDLE \((\w+)\)/g;
+const downloadEndRegex = /END DL \((\w+)\)/g;
+const downloadRegex = /DL \((\w+)\) \((\d+)\/(\d+)\)/g;
+const processingRegex = /PROCESS \((\d+)\/(\d+)\)/g;
+
+export function useDatasetDownloadManager() {
+	const [managerState, setManagerState] = useState<DownloadManagerStatus>(DownloadManagerStatus.IDLE);
+	const [datasetStates, setDatasetStates] = useState<Record<string, DatasetStatus>>({});
+	const [processingState, setProcessingState] = useState<DatasetStatus | undefined>(undefined);
+	const [idle, setIdle] = useState<boolean>(true);
+	const [worker, setWorker] = useState<Worker>();
+
+	const startDownload = useCallback(
+		idle
+			? () => {
+				worker?.postMessage({ type: 'loadDepsData' });
+				setProcessingState({ name: 'Processing downloaded files', status: DatasetDownloadStatus.PROCESSING, done: 0, total: 1 });
+				setManagerState(DownloadManagerStatus.WORKING);
+				setIdle(false);
+			}
+			: () => {},
+		[worker, idle],
+	);
+
+	useEffect(() => {
+		setWorker(new Worker(new URL('./geantWorker.worker.ts', import.meta.url)));
+	}, []);
+
+	useEffect(() => {
+		let done = '', total = '', dataset = '';
+		const handler = (event: MessageEvent) => {
+			switch (event.data.type) {
+				case 'status':
+					switch (true) {
+						case event.data.data?.startsWith('IDLE'):
+							[, dataset] = Array.from(event.data.data.matchAll(idleRegex))[0] as string[];
+							setDatasetStates(states => ({
+								...states,
+								[dataset]: {
+									name: dataset,
+									status: states[dataset] ? states[dataset].status : DatasetDownloadStatus.IDLE,
+								}
+							}));
+							break;
+						case event.data.data?.startsWith('DL'):
+							[, dataset, done, total] = Array.from(event.data.data.matchAll(downloadRegex))[0] as string[];
+							setDatasetStates(states => ({
+								...states,
+								[dataset]: {
+									name: dataset,
+									status: DatasetDownloadStatus.DOWNLOADING,
+									done: parseInt(done),
+									total: parseInt(total)
+								}
+							}));
+							break;
+						case event.data.data?.startsWith('END DL'):
+							[, dataset] = Array.from(event.data.data.matchAll(downloadEndRegex))[0] as string[];
+							console.log('end dl', dataset);
+							setDatasetStates(states => ({
+								...states,
+								[dataset]: {
+									name: dataset,
+									status: DatasetDownloadStatus.DONE,
+									done: parseInt(done),
+									total: parseInt(total)
+								}
+							}));
+							break;
+						case event.data.data?.startsWith('PROCESS'):
+							[, done, total] = Array.from(event.data.data.matchAll(processingRegex))[0] as string[];
+							setProcessingState({
+								name: 'Processing downloaded files',
+								status: DatasetDownloadStatus.PROCESSING,
+								done: parseInt(done),
+								total: parseInt(total)
+							})
+							break;
+						case event.data.data?.startsWith('INIT END'):
+							setManagerState(DownloadManagerStatus.FINISHED);
+							setDatasetStates(states => Object.fromEntries(Object.entries(states).map(([k, v], _) => [k, { ...v, status: DatasetDownloadStatus.DONE }])));
+							setProcessingState({ name: 'Processing downloaded files', status: DatasetDownloadStatus.DONE });
+							break;
+					}
+					break;
+				case 'error':
+					setManagerState(DownloadManagerStatus.ERROR);
+					break;
+			}
+		};
+		worker?.addEventListener('message', handler);
+		return () => worker?.removeEventListener('message', handler);
+	}, [worker]);
+
+	let allStates = Object.values(datasetStates);
+	if (processingState) {
+		allStates.push(processingState);
+	}
+
+	return { managerState, datasetStates: allStates, startDownload };
+}

src/libs/geant4_web/GeantWorkerInterface.ts

@@ -0,0 +1,19 @@
+
+export enum GeantWorkerMessageType {
+    INIT_DATA_FILES,
+    INIT_LAZY_FILES,
+    CREATE_FILE,
+    READ_FILE,
+    RUN_SIMULATION,
+    FILE_RESPONSE
+}
+
+export type GeantWorkerMessageFile = {
+    name: string,
+    data: string 
+}
+
+export type GeantWorkerMessage = {
+    type: GeantWorkerMessageType,
+    data: GeantWorkerMessageFile | string
+}