From e7ec1564d702ebbd5ea39e715440a9421f5ff535 Mon Sep 17 00:00:00 2001 From: Rehpotsirhc <86068565+Rehpotsirhc-z@users.noreply.github.com> Date: Mon, 8 Jun 2026 15:43:38 -0400 Subject: [PATCH 1/6] Add remote backend --- backend/funcs/get_data.py | 381 ++++++++---------- backend/funcs/remote_io.py | 237 +++++++++++ backend/main.py | 9 + backend/requirements.txt | 3 +- backend/routes/qtl_routes.py | 7 + backend/settings.py | 4 + frontend/src/api/qtl.js | 12 + .../src/pages/Datasets/DatasetDisplay.jsx | 20 +- .../Datasets/LoadRemoteDatasetDialog.jsx | 223 ++++++++++ frontend/src/pages/Datasets/index.jsx | 9 + frontend/src/store/DatatableStore.js | 13 +- frontend/src/store/RemoteDatasetStore.js | 64 +++ 12 files changed, 760 insertions(+), 222 deletions(-) create mode 100644 backend/funcs/remote_io.py create mode 100644 frontend/src/pages/Datasets/LoadRemoteDatasetDialog.jsx create mode 100644 frontend/src/store/RemoteDatasetStore.js diff --git a/backend/funcs/get_data.py b/backend/funcs/get_data.py index 34ea6e8..cc55437 100644 --- a/backend/funcs/get_data.py +++ b/backend/funcs/get_data.py @@ -9,6 +9,17 @@ import re from functools import lru_cache +from backend.funcs.remote_io import ( + is_remote, + ds_locator, + ds_exists, + ds_read_json, + ds_read_text, + ds_read_toml, + ds_text_stream, + ds_bytes_stream, +) + def safe_filename(name): return re.sub(r"[^a-zA-Z0-9_\-]", "_", name) @@ -64,22 +75,17 @@ def get_gene_location(dataset, gene): return f"Error: Could not parse positions from gene {gene}: {str(e)}" # Case 2: eQTL dataset - need to look up in JSON file - genes_file = os.path.join( - "backend", "datasets", dataset, "gene_jsons", - get_gene_file_name(safe_filename(gene)) + ".json" - ) + gene_file = get_gene_file_name(safe_filename(gene)) + ".json" + data = ds_read_json(dataset, "gene_jsons", gene_file) - if not os.path.exists(genes_file): - print(genes_file + " not found") + if data is None: + print(f"gene_jsons/{gene_file} not found for {dataset}") return "Error: Gene list file not found for the specified dataset." - - with open(genes_file, "r") as f: - data = json.load(f) - - if isinstance(data, dict) and gene in data: - gene_x = data[gene] - else: - gene_x = data + + if isinstance(data, dict) and gene in data: + gene_x = data[gene] + else: + gene_x = data if not gene_x: return f"Error: Gene {gene} not found in dataset." @@ -105,19 +111,10 @@ def get_snp_location(dataset, snp): if dataset == "all": return "Error: Dataset is not specified." - def load_snp_from_file(file_path): - if os.path.exists(file_path): - with open(file_path, "r") as f: - data = json.load(f) - return data.get(snp, None) - return None - try: group = get_snp_group(snp) - snps_file = os.path.join( - "backend", "datasets", dataset, "snp_jsons_merged", group + ".json" - ) - snp_data = load_snp_from_file(snps_file) + group_map = ds_read_json(dataset, "snp_jsons_merged", group + ".json") + snp_data = group_map.get(snp) if isinstance(group_map, dict) else None if snp_data: return { "chromosome": snp_data["chromosome"], @@ -128,12 +125,14 @@ def load_snp_from_file(file_path): pass default_file = os.path.join("backend", "datasets", "snp_locations.json") - snp_data = load_snp_from_file(default_file) - if snp_data: - return { - "chromosome": snp_data["chromosome"], - "position": snp_data["position"], - } + if os.path.exists(default_file): + with open(default_file, "r") as f: + snp_data = json.load(f).get(snp, None) + if snp_data: + return { + "chromosome": snp_data["chromosome"], + "position": snp_data["position"], + } return None @@ -142,13 +141,11 @@ def get_gene_locations_in_chromosome(dataset, chromosome, start, end): if dataset == "all": return "Error: Dataset is not specified." else: - chromosome_file = os.path.join( - "backend", "datasets", dataset, "gene_locations", chromosome + ".parquet" - ) + rel = ("gene_locations", chromosome + ".parquet") # significant_genes_list = get_qtl_gene_list(dataset) - if os.path.exists(chromosome_file): - df = pl.read_parquet(chromosome_file).filter( + if ds_exists(dataset, *rel): + df = pl.read_parquet(ds_locator(dataset, *rel)).filter( (pl.col("position_end") >= start) & (pl.col("position_start") <= end) # & pl.col("gene_id").is_in(significant_genes_list) @@ -161,7 +158,7 @@ def get_gene_locations_in_chromosome(dataset, chromosome, start, end): else: return f"No genes found in the region." else: - print(chromosome_file + " not found") + print(f"gene_locations/{chromosome}.parquet not found for {dataset}") return "Error: Chromosome file not found for the specified dataset." @@ -169,13 +166,11 @@ def get_snp_locations_in_chromosome(dataset, chromosome, start, end): if dataset == "all": return "Error: Dataset is not specified." else: - chromosome_file = os.path.join( - "backend", "datasets", dataset, "snp_locations", chromosome + ".parquet" - ) + rel = ("snp_locations", chromosome + ".parquet") significant_snps_list = get_qtl_snp_list(dataset) - if os.path.exists(chromosome_file): - df = pl.read_parquet(chromosome_file).filter( + if ds_exists(dataset, *rel): + df = pl.read_parquet(ds_locator(dataset, *rel)).filter( (pl.col("position") >= start) & (pl.col("position") <= end) & pl.col("snp_id").is_in(significant_snps_list) @@ -186,31 +181,28 @@ def get_snp_locations_in_chromosome(dataset, chromosome, start, end): else: return f"No SNPs found in the region." else: - print(chromosome_file + " not found") + print(f"snp_locations/{chromosome}.parquet not found for {dataset}") return "Error: Chromosome file not found for the specified dataset." def get_gwas_datasets(dataset): - config_path = os.path.join("backend", "datasets", dataset, "gwas", "gwas_datasets.toml") - if not os.path.exists(config_path): - return [] - with open(config_path, "r") as f: - config = toml.load(f) + config = ds_read_toml(dataset, "gwas", "gwas_datasets.toml") + if config is None: + return [] return config.get("datasets", []) def get_gwas_in_chromosome(qtl_dataset, gwas_dataset, chromosome, start, end): - base_dir = os.path.join("backend", "datasets", qtl_dataset, "gwas", gwas_dataset) - file_path = os.path.join(base_dir, f"{chromosome}.tsv") - if not os.path.exists(file_path): - # Return empty - return { - "snp_id": [], - "position": [], - "beta_value": [], - "p_value": [], - } - df = pl.read_csv(file_path, separator="\t").filter( + empty = { + "snp_id": [], + "position": [], + "beta_value": [], + "p_value": [], + } + tsv_stream = ds_bytes_stream(qtl_dataset, "gwas", gwas_dataset, f"{chromosome}.tsv") + if tsv_stream is None: + return empty + df = pl.read_csv(tsv_stream, separator="\t").filter( (pl.col("position") >= start) & (pl.col("position") <= end) ) if df.is_empty(): @@ -234,16 +226,10 @@ def get_gene_chromosome(dataset, gene): chromosome = gene.split("-")[0] return chromosome - genes_file = os.path.join( - "backend", "datasets", dataset, "gene_jsons", - get_gene_file_name(safe_filename(gene)) + ".json" - ) - - if os.path.exists(genes_file): - with open(genes_file, "r") as f: - data = json.load(f) - gene_x = data + gene_file = get_gene_file_name(safe_filename(gene)) + ".json" + gene_x = ds_read_json(dataset, "gene_jsons", gene_file) + if gene_x is not None: if gene_x: chromosome = gene_x.get("chromosome") if chromosome: @@ -253,7 +239,7 @@ def get_gene_chromosome(dataset, gene): else: return f"Error: Gene {gene} not found in {dataset} dataset." else: - print(genes_file + " not found") + print(f"gene_jsons/{gene_file} not found for {dataset}") return f"Error: Gene list file not found for the specified dataset {dataset}." @@ -261,32 +247,28 @@ def get_snp_chromosome(dataset, snp): if dataset == "all": return "Error: Dataset is not specified." else: - snps_file = os.path.join( - "backend", "datasets", dataset, "snp_jsons_merged", get_snp_group(snp) + ".json" - ) + snps_file = ("snp_jsons_merged", get_snp_group(snp) + ".json") - if os.path.exists(snps_file): - with open(snps_file, "r") as f: - data = json.load(f) - snp = data.get(snp, None) + data = ds_read_json(dataset, *snps_file) + if data is not None: + snp = data.get(snp, None) if data and snp: return snp["chromosome"] else: return f"Error: SNP {snp} not found in {dataset} dataset." else: - print(snps_file + " not found") + print(f"{'/'.join(snps_file)} not found for {dataset}") return "Error: SNP list file not found for the specified dataset." def get_gene_list(dataset, query_str="AB"): if dataset == "all": genes_file = os.path.join("backend", "datasets", "gene_list.json") + data = json.load(open(genes_file)) if os.path.exists(genes_file) else None else: - genes_file = os.path.join("backend", "datasets", dataset, "gene_list.json") + data = ds_read_json(dataset, "gene_list.json") - if os.path.exists(genes_file): - with open(genes_file, "r") as f: - data = json.load(f) + if data is not None: if query_str == "all": return data elif query_str == "default": @@ -294,22 +276,16 @@ def get_gene_list(dataset, query_str="AB"): else: return [gene for gene in data if gene.lower().startswith(query_str.lower())] else: - print(genes_file + " not found") + print(f"gene_list.json not found for {dataset}") return "Error: Gene list file not found" def get_qtl_gene_list(dataset, query_str="all"): if dataset == "all": return "Error: Gene dataset not specified." - else: - genes_file = os.path.join( - "backend", "datasets", dataset, "gene_list.json" - ) - - if os.path.exists(genes_file): - with open(genes_file, "r") as f: - data = json.load(f) + data = ds_read_json(dataset, "gene_list.json") + if data is not None: data = _normalize_gene_list(data) if query_str == "all": @@ -329,72 +305,62 @@ def get_qtl_gene_list(dataset, query_str="all"): "name": [g["name"] for g in filtered], } else: - print(genes_file + " not found") + print(f"gene_list.json not found for {dataset}") return "Error: Gene list file not found" def get_qtl_snp_list(dataset, query_str="all"): if dataset == "all": return "Error: SNP dataset not specified." - else: - snps_file = os.path.join("backend", "datasets", dataset, "snp_list.json") - if os.path.exists(snps_file): - with open(snps_file, "r") as f: - data = json.load(f) + data = ds_read_json(dataset, "snp_list.json") + if data is not None: if query_str == "all": return data else: return [snp for snp in data if snp.lower().startswith(query_str.lower())] else: - print(snps_file + " not found") + print(f"snp_list.json not found for {dataset}") return "Error: SNP list file not found" def get_gene_celltypes(dataset, gene): if dataset == "all": return "Error: Dataset is not specified." - else: - genes_file = os.path.join( - "backend", "datasets", dataset, "gene_jsons", get_gene_file_name(safe_filename(gene)) + ".json" - ) - if os.path.exists(genes_file): - with open(genes_file, "r") as f: - data = json.load(f) + gene_file = get_gene_file_name(safe_filename(gene)) + ".json" + data = ds_read_json(dataset, "gene_jsons", gene_file) - if gene.startswith("chr"): - gene_x = data.get(gene, None) - else: - gene_x = data + if data is not None: + if gene.startswith("chr"): + gene_x = data.get(gene, None) + else: + gene_x = data if gene_x: return gene_x.get("celltypes", []) else: return f"Error: Gene not found in dataset." else: - print(genes_file + " not found") + print(f"gene_jsons/{gene_file} not found for {dataset}") return "Error: Gene list file not found for the specified dataset." def get_snp_celltypes(dataset, snp): if dataset == "all": return "Error: Dataset is not specified." - else: - snps_file = os.path.join( - "backend", "datasets", dataset, "snp_jsons_merged", get_snp_group(snp) + ".json" - ) - if os.path.exists(snps_file): - with open(snps_file, "r") as f: - data = json.load(f) - snp = data.get(snp, None) + snps_file = ("snp_jsons_merged", get_snp_group(snp) + ".json") + data = ds_read_json(dataset, *snps_file) + + if data is not None: + snp = data.get(snp, None) if data and snp: return snp["celltypes"] else: return f"Error: SNP not found in dataset." else: - print(snps_file + " not found") + print(f"{'/'.join(snps_file)} not found for {dataset}") return "Error: SNP list file not found for the specified dataset." @@ -411,22 +377,16 @@ def get_snp_data_for_gene(dataset, gene, celltype=""): gene_start = gene_loc["start"] gene_end = gene_loc["end"] - celltype_mapping_file = os.path.join( - "backend", "datasets", dataset, "celltypes", "celltype_parquet.json" - ) - if not os.path.exists(celltype_mapping_file): - print(celltype_mapping_file + " not found") + celltype_mapping = ds_read_json(dataset, "celltypes", "celltype_parquet.json") + if celltype_mapping is None: + print(f"celltypes/celltype_parquet.json not found for {dataset}") return "Error: Celltype mapping file not found for the specified dataset." - with open(celltype_mapping_file, "r") as f: - celltype_mapping = json.load(f) - celltype_file = celltype_mapping.get(celltype, celltype) - data_file = os.path.join("backend", "datasets", dataset, "celltypes", celltype_file) - - if not os.path.exists(data_file): - print(data_file + " not found") + if not ds_exists(dataset, "celltypes", celltype_file): + print(f"celltypes/{celltype_file} not found for {dataset}") return "Error: QTL data file not found for the specified dataset and cell type." + data_file = ds_locator(dataset, "celltypes", celltype_file) gene_df = ( pl.scan_parquet(data_file) @@ -475,22 +435,16 @@ def get_gene_data_for_snp(dataset, is_caqtl, snp, celltype=""): return f"Error: {snp_loc}" chrom = snp_loc["chromosome"] - celltype_mapping_file = os.path.join( - "backend", "datasets", dataset, "celltypes", "celltype_parquet.json" - ) - if not os.path.exists(celltype_mapping_file): - print(celltype_mapping_file + " not found") + celltype_mapping = ds_read_json(dataset, "celltypes", "celltype_parquet.json") + if celltype_mapping is None: + print(f"celltypes/celltype_parquet.json not found for {dataset}") return "Error: Celltype mapping file not found for the specified dataset." - with open(celltype_mapping_file, "r") as f: - celltype_mapping = json.load(f) - celltype_file = celltype_mapping.get(celltype, celltype) - data_file = os.path.join("backend", "datasets", dataset, "celltypes", celltype_file) - - if not os.path.exists(data_file): - print(data_file + " not found") + if not ds_exists(dataset, "celltypes", celltype_file): + print(f"celltypes/{celltype_file} not found for {dataset}") return "Error: QTL data file not found for the specified dataset and cell type." + data_file = ds_locator(dataset, "celltypes", celltype_file) snp_df = ( pl.scan_parquet(data_file) @@ -528,14 +482,11 @@ def parse_region(region: str): ) return snp_df.to_dict(as_series=False) - gene_loc_file = os.path.join( - "backend", "datasets", dataset, "gene_locations", f"{chrom}.parquet" - ) - if not os.path.exists(gene_loc_file): - print(gene_loc_file + " not found") + if not ds_exists(dataset, "gene_locations", f"{chrom}.parquet"): + print(f"gene_locations/{chrom}.parquet not found for {dataset}") return "Error: Gene locations file not found for the specified dataset." - gene_loc_df = pl.read_parquet(gene_loc_file) + gene_loc_df = pl.read_parquet(ds_locator(dataset, "gene_locations", f"{chrom}.parquet")) if "gene_name" not in gene_loc_df.columns: return "Error: gene_locations parquet has no gene_name column for eQTL dataset." @@ -623,15 +574,12 @@ def get_meta_list(dataset, query_str="all"): def get_config_info(dataset): if dataset == "all": return "Error: Dataset is not specified." - else: - config_file = os.path.join("backend", "datasets", dataset, "dataset_info.toml") - if os.path.exists(config_file): - with open(config_file, "r") as f: - config = toml.load(f) + config = ds_read_toml(dataset, "dataset_info.toml") + if config is not None: return config else: - print(config_file + " not found") + print(f"dataset_info.toml not found for {dataset}") return "Error: Config info file not found" @@ -1040,6 +988,11 @@ def get_bw_data_exists(dataset): if dataset == "all": return "Error: Dataset is not specified." + # For remote datasets there is no listable folder; we use the bigwig + # celltype mapping file to determine if signal tracks are available + if is_remote(dataset): + return ds_exists(dataset, "bigwig", "celltype_bigwig.json") + bw_folder = os.path.join("backend", "datasets", dataset,"bigwig") if not os.path.exists(bw_folder): return False @@ -1047,16 +1000,11 @@ def get_bw_data_exists(dataset): @lru_cache(maxsize=128) def get_cached_bigwig_handle(dataset, celltype, strand=None): - celltype_mapping_file = os.path.join( - "backend", "datasets", dataset,"bigwig","celltype_bigwig.json" - ) - if not os.path.exists(celltype_mapping_file): - print(celltype_mapping_file + " not found") + celltype_mapping = ds_read_json(dataset, "bigwig", "celltype_bigwig.json") + if celltype_mapping is None: + print(f"bigwig/celltype_bigwig.json not found for {dataset}") return None - with open(celltype_mapping_file, "r") as f: - celltype_mapping = json.load(f) - entry = celltype_mapping.get(celltype, celltype) if isinstance(entry, str): @@ -1073,13 +1021,7 @@ def get_cached_bigwig_handle(dataset, celltype, strand=None): print(f"No matching bigWig for celltype={celltype}, strand={strand}") return None - bw_path = os.path.join( - "backend", - "datasets", - dataset, - "bigwig", - celltype_file, - ) + bw_path = ds_locator(dataset, "bigwig", celltype_file) try: return pyBigWig.open(bw_path) @@ -1110,16 +1052,11 @@ def get_region_signal_data(dataset, chromosome, start, end, celltype="", bin_siz try: # Read mapping once here to know if we have plus/minus - celltype_mapping_file = os.path.join( - "backend", "datasets", dataset, "bigwig", "celltype_bigwig.json" - ) - if not os.path.exists(celltype_mapping_file): - print(celltype_mapping_file + " not found") + celltype_mapping = ds_read_json(dataset, "bigwig", "celltype_bigwig.json") + if celltype_mapping is None: + print(f"bigwig/celltype_bigwig.json not found for {dataset}") return "Error: BigWig mapping file not found" - with open(celltype_mapping_file, "r") as f: - celltype_mapping = json.load(f) - entry = celltype_mapping.get(celltype, celltype) if strand in {"+", "-"}: @@ -1270,19 +1207,12 @@ def get_gene_annotation_info(dataset): if dataset == "all": return {} - ann_file = os.path.join( - "backend", "datasets", dataset, "gene_locations", "gene_annotations.toml" - ) - if not os.path.exists(ann_file): + try: + cfg = ds_read_toml(dataset, "gene_locations", "gene_annotations.toml") + except Exception as e: + print(f"Error reading gene_annotations.toml for {dataset}: {e}") return {} - with open(ann_file, "r") as f: - try: - cfg = toml.load(f) - except Exception as e: - print(f"Error reading {ann_file}: {e}") - return {} - return cfg or {} @@ -1297,16 +1227,11 @@ def get_celltype_list(dataset): if dataset == "all": return "Error: Dataset is not specified." - celltype_mapping_file = os.path.join( - "backend", "datasets", dataset,'celltypes', "celltype_parquet.json" - ) - if not os.path.exists(celltype_mapping_file): - print(celltype_mapping_file + " not found") + celltype_mapping = ds_read_json(dataset, "celltypes", "celltype_parquet.json") + if celltype_mapping is None: + print(f"celltypes/celltype_parquet.json not found for {dataset}") return f"Error: Celltype mapping file not found for the dataset" - with open(celltype_mapping_file, "r") as f: - celltype_mapping = json.load(f) - return list(celltype_mapping.keys()) @@ -1314,19 +1239,59 @@ def get_bigwig_celltype_list(dataset): if dataset == "all": return "Error: Dataset is not specified." - celltype_mapping_file = os.path.join( - "backend", "datasets", dataset, "bigwig", "celltype_bigwig.json" - ) - if not os.path.exists(celltype_mapping_file): - print(celltype_mapping_file + " not found") + celltype_mapping = ds_read_json(dataset, "bigwig", "celltype_bigwig.json") + if celltype_mapping is None: + print(f"bigwig/celltype_bigwig.json not found for {dataset}") return "Error: BigWig celltype mapping file not found for the dataset" - with open(celltype_mapping_file, "r") as f: - celltype_mapping = json.load(f) - return list(celltype_mapping.keys()) +def inspect_dataset(dataset): + if not dataset or dataset == "all": + return {"reachable": False, "error": "No dataset specified."} + + try: + has_qtl = bool(ds_exists(dataset, "celltypes", "celltype_parquet.json")) + has_bw = get_bw_data_exists(dataset) + has_bw = bool(has_bw) if not isinstance(has_bw, str) else False + + config = get_config_info(dataset) + datatype = "" + info = {} + if isinstance(config, dict): + datatype = (config.get("datasetfile", {}) or {}).get("datatype", "") or "" + info = config.get("dataset", {}) or {} + + is_caqtl = datatype.lower().startswith("ca") + + # Fall back to structural heuristics when there is no dataset_info.toml. + if not datatype: + if has_qtl: + # caQTL groups gene jsons by chromosome (chr1.json ...), eQTL uses + # one file per gene symbol. + is_caqtl = bool(ds_exists(dataset, "gene_jsons", "chr1.json")) + datatype = "caQTL" if is_caqtl else "eQTL" + elif has_bw: + datatype = "scATACseq" + + reachable = bool(has_qtl or has_bw or isinstance(config, dict)) + + return { + "reachable": reachable, + "dataset_id": dataset, + "assay": datatype, + "is_caqtl": is_caqtl, + "has_qtl": has_qtl, + "has_bw": has_bw, + "dataset_name": info.get("dataset_name", "") if info else "", + "info": info, + } + except Exception as e: + # Includes RemoteDatasetError (unsafe/blocked URL) and transport errors + return {"reachable": False, "dataset_id": dataset, "error": str(e)} + + def get_exon_structure_in_chromosome(dataset, chromosome, start, end, max_exons=2000): if dataset == "all": return "Error: Dataset is not specified." @@ -1341,9 +1306,9 @@ def get_exon_structure_in_chromosome(dataset, chromosome, start, end, max_exons= # return "Error: Annotation BED file not specified for the dataset." bed_rel = "transcript_annotation.bed" - bed_path = os.path.join("backend", "datasets", dataset, bed_rel) - if not os.path.exists(bed_path): - print(bed_path + " not found") + bed_stream = ds_text_stream(dataset, bed_rel) + if bed_stream is None: + print(f"{bed_rel} not found for {dataset}") return "Error: Annotation BED file not found for the specified dataset." chrom_list = [] @@ -1363,7 +1328,7 @@ def get_exon_structure_in_chromosome(dataset, chromosome, start, end, max_exons= truncated = False import csv - with open(bed_path) as f: + with bed_stream as f: reader = csv.reader(f, delimiter="\t") for row in reader: if len(row) < 12: diff --git a/backend/funcs/remote_io.py b/backend/funcs/remote_io.py new file mode 100644 index 0000000..0b32c6b --- /dev/null +++ b/backend/funcs/remote_io.py @@ -0,0 +1,237 @@ +import io +import ipaddress +import json +import os +import socket +from functools import lru_cache +from urllib.parse import urljoin, urlsplit, quote + +import requests +import toml + +from backend.settings import settings + +DATASETS_DIR = os.path.join("backend", "datasets") + +# Networking limits for remote fetches. +REMOTE_TIMEOUT = 30 # seconds per request +MAX_REMOTE_BYTES = 1024 * 1024 * 1024 # 1 GiB cap for whole-file downloads +MAX_REDIRECTS = 3 + + +class RemoteDatasetError(Exception): + pass + + +def is_remote(dataset) -> bool: + return isinstance(dataset, str) and ( + dataset.startswith("http://") or dataset.startswith("https://") + ) + + +# --------------------------------------------------------------------------- # +# SSRF protection +# --------------------------------------------------------------------------- # +def _allowed_hosts() -> set: + raw = getattr(settings, "remote_dataset_allowed_hosts", "") or "" + return {h.strip().lower() for h in raw.split(",") if h.strip()} + + +def _ip_is_public(ip_text: str) -> bool: + try: + ip = ipaddress.ip_address(ip_text) + except ValueError: + return False + + # Unwrap IPv4-mapped / 6to4-style addresses so 127.0.0.1 can't be hidden as ::ffff:127.0.0.1 + if ip.version == 6 and getattr(ip, "ipv4_mapped", None) is not None: + ip = ip.ipv4_mapped + if ( + ip.is_private + or ip.is_loopback + or ip.is_link_local + or ip.is_multicast + or ip.is_reserved + or ip.is_unspecified + ): + return False + return bool(ip.is_global) + + +def _resolve_ips(host: str) -> set: + infos = socket.getaddrinfo(host, None) + return {info[4][0] for info in infos} + + +def assert_safe_url(url: str) -> str: + if not getattr(settings, "allow_remote_datasets", True): + raise RemoteDatasetError("Remote datasets are disabled on this server.") + + parts = urlsplit(url) + if parts.scheme not in ("http", "https"): + raise RemoteDatasetError(f"Unsupported URL scheme: {parts.scheme!r}") + host = parts.hostname + if not host: + raise RemoteDatasetError("Remote dataset URL has no host.") + + allow = _allowed_hosts() + if allow and host.lower() not in allow: + raise RemoteDatasetError(f"Host '{host}' is not in the allowed-hosts list.") + + if getattr(settings, "remote_dataset_allow_private", False): + return url + + try: + ips = _resolve_ips(host) + except socket.gaierror as exc: + raise RemoteDatasetError(f"Could not resolve host '{host}': {exc}") + if not ips: + raise RemoteDatasetError(f"Could not resolve host '{host}'.") + for ip in ips: + if not _ip_is_public(ip): + raise RemoteDatasetError( + f"Refusing to access non-public address {ip} (host '{host}')." + ) + return url + + +# --------------------------------------------------------------------------- # +# HTTP helpers +# --------------------------------------------------------------------------- # +def _request(method: str, url: str, **kwargs): + """Issue a request, re-validating the target on every redirect hop.""" + assert_safe_url(url) + current = url + for _ in range(MAX_REDIRECTS + 1): + resp = requests.request( + method, + current, + allow_redirects=False, + timeout=REMOTE_TIMEOUT, + **kwargs, + ) + if resp.status_code in (301, 302, 303, 307, 308): + location = resp.headers.get("Location") + resp.close() + if not location: + raise RemoteDatasetError("Redirect response without a Location header.") + current = urljoin(current, location) + assert_safe_url(current) + continue + return resp + raise RemoteDatasetError("Too many redirects while fetching remote dataset file.") + + +def _remote_exists(url: str) -> bool: + try: + resp = _request("GET", url, stream=True, headers={"Range": "bytes=0-0"}) + except RemoteDatasetError: + raise + except requests.RequestException: + return False + try: + return resp.status_code < 400 + finally: + resp.close() + + +@lru_cache(maxsize=64) +def _fetch_bytes_cached(url: str) -> bytes: + try: + resp = _request("GET", url, stream=True) + except requests.RequestException as exc: + raise RemoteDatasetError(f"Failed to fetch {url}: {exc}") + + with resp: + if resp.status_code == 404: + raise FileNotFoundError(url) + if resp.status_code >= 400: + raise RemoteDatasetError( + f"Remote returned HTTP {resp.status_code} for {url}" + ) + + total = 0 + chunks = [] + for chunk in resp.iter_content(chunk_size=1 << 16): + if not chunk: + continue + total += len(chunk) + if total > MAX_REMOTE_BYTES: + raise RemoteDatasetError( + f"Remote file {url} exceeds the {MAX_REMOTE_BYTES} byte limit." + ) + chunks.append(chunk) + return b"".join(chunks) + + +# --------------------------------------------------------------------------- # +# Path / URL building +# --------------------------------------------------------------------------- # +def _local_path(dataset: str, parts) -> str: + return os.path.join(DATASETS_DIR, dataset, *parts) + + +def _remote_url(dataset: str, parts) -> str: + base = dataset.rstrip("/") + suffix = "/".join(quote(str(p), safe="") for p in parts) + return f"{base}/{suffix}" if suffix else base + + +def ds_locator(dataset: str, *parts) -> str: + if is_remote(dataset): + return assert_safe_url(_remote_url(dataset, parts)) + return _local_path(dataset, parts) + + +def ds_exists(dataset: str, *parts) -> bool: + if is_remote(dataset): + return _remote_exists(_remote_url(dataset, parts)) + return os.path.exists(_local_path(dataset, parts)) + + +def ds_read_bytes(dataset: str, *parts): + if is_remote(dataset): + try: + return _fetch_bytes_cached(_remote_url(dataset, parts)) + except FileNotFoundError: + return None + path = _local_path(dataset, parts) + if not os.path.exists(path): + return None + with open(path, "rb") as fh: + return fh.read() + + +def ds_read_text(dataset: str, *parts, encoding="utf-8"): + data = ds_read_bytes(dataset, *parts) + if data is None: + return None + return data.decode(encoding) + + +def ds_read_json(dataset: str, *parts): + text = ds_read_text(dataset, *parts) + if text is None: + return None + return json.loads(text) + + +def ds_read_toml(dataset: str, *parts): + text = ds_read_text(dataset, *parts) + if text is None: + return None + return toml.loads(text) + + +def ds_text_stream(dataset: str, *parts): + text = ds_read_text(dataset, *parts) + if text is None: + return None + return io.StringIO(text) + + +def ds_bytes_stream(dataset: str, *parts): + data = ds_read_bytes(dataset, *parts) + if data is None: + return None + return io.BytesIO(data) diff --git a/backend/main.py b/backend/main.py index 3857b2a..e413502 100644 --- a/backend/main.py +++ b/backend/main.py @@ -1,13 +1,22 @@ from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse +from starlette.requests import Request from backend.db import create_db_and_tables from backend.routes import db_routes, api_routes, visium_routes, qtl_routes, dm_routes,signal_routes +from backend.funcs.remote_io import RemoteDatasetError from backend.settings import settings app = FastAPI(debug=settings.debug) + +@app.exception_handler(RemoteDatasetError) +async def remote_dataset_error_handler(request: Request, exc: RemoteDatasetError): + # A remote (client-supplied) dataset URL was disallowed or unreachable. + return JSONResponse(status_code=400, content={"detail": f"Remote dataset error: {exc}"}) + app.add_middleware( CORSMiddleware, allow_origins=[ diff --git a/backend/requirements.txt b/backend/requirements.txt index c9e7a88..c0d0966 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -11,5 +11,6 @@ polars == 1.31.0 fastparquet == 2024.11.0 types-pytz == 2025.2.* pydantic-settings == 2.10.1 -pyBigWig == 0.3.24 +pyBigWig == 0.3.24 # pyBigWig must be built against libcurl so it can stream remote (http/https) +requests == 2.32.* diff --git a/backend/routes/qtl_routes.py b/backend/routes/qtl_routes.py index 3bcc1f5..c377ffd 100644 --- a/backend/routes/qtl_routes.py +++ b/backend/routes/qtl_routes.py @@ -19,6 +19,7 @@ get_gwas_datasets, get_gwas_in_chromosome, get_gencode_version, + inspect_dataset, ) router = APIRouter() @@ -29,6 +30,12 @@ async def read_root(): return {"Message": "Hello QTL."} +@router.get("/inspectdataset") +async def inspectdataset(request: Request): + dataset_id = request.query_params.get("dataset") + return inspect_dataset(dataset_id) + + @router.get("/getgenelocation") async def getgenelocation(request: Request): print("getgenelocation() called================") diff --git a/backend/settings.py b/backend/settings.py index 74abff7..235a0dd 100644 --- a/backend/settings.py +++ b/backend/settings.py @@ -5,6 +5,10 @@ class Settings(BaseSettings): uvicorn_host: str = "0.0.0.0" debug: bool = False + allow_remote_datasets: bool = True + remote_dataset_allowed_hosts: str = "" + remote_dataset_allow_private: bool = False + class Config: env_file = ".env" # 可选:已手动 load_dotenv 也可以省略 diff --git a/frontend/src/api/qtl.js b/frontend/src/api/qtl.js index c57a0b9..e083518 100644 --- a/frontend/src/api/qtl.js +++ b/frontend/src/api/qtl.js @@ -3,6 +3,18 @@ import axios from "axios"; const BASE_URL = import.meta.env.VITE_BACKEND_URL; const QTL_URL = `${BASE_URL}/qtl`; +export const inspectDataset = async (dataset) => { + try { + const response = await axios.get(`${QTL_URL}/inspectdataset`, { + params: { dataset: dataset }, + }); + return response.data; + } catch (error) { + console.error("Error inspectDataset:", error); + throw error; + } +}; + export const getGeneLocation = async (dataset, gene) => { try { const response = await axios.get(`${QTL_URL}/getgenelocation`, { diff --git a/frontend/src/pages/Datasets/DatasetDisplay.jsx b/frontend/src/pages/Datasets/DatasetDisplay.jsx index aec1059..8165199 100644 --- a/frontend/src/pages/Datasets/DatasetDisplay.jsx +++ b/frontend/src/pages/Datasets/DatasetDisplay.jsx @@ -218,10 +218,10 @@ const DatasetDisplay = ({dataRecords, deleteMode}) => { {record.assay} - {["scrnaseq", "snrnaseq", "visiumst","merfish"].includes(record.assay.toLowerCase()) && (UMAP)} - {/*{["visiumst","visium"].includes(record.assay.toLowerCase()) && (Visium)}*/} - {["eqtl", "caqtl"].includes(record.assay.toLowerCase()) && (xQTL)} - {record.has_bw && (Peaks)} + {["scrnaseq", "snrnaseq", "visiumst","merfish"].includes(record.assay.toLowerCase()) && (UMAP)} + {/*{["visiumst","visium"].includes(record.assay.toLowerCase()) && (Visium)}*/} + {["eqtl", "caqtl"].includes(record.assay.toLowerCase()) && (xQTL)} + {record.has_bw && (Peaks)} {deleteMode && ( @@ -242,7 +242,9 @@ const DatasetDisplay = ({dataRecords, deleteMode}) => { {displayedData.map((record) => ( - {record.dataset_id} + {record.is_remote || record.sample_sheet === "None" || record.sample_sheet === null || (record.sample_sheet ?? "").trim() === "" ? + (record.dataset_name || record.dataset_id) : + {record.dataset_id}} @@ -262,10 +264,10 @@ const DatasetDisplay = ({dataRecords, deleteMode}) => { - {["scrnaseq", "snrnaseq", "visiumst", "merfish"].includes(record.assay.toLowerCase()) && (UMAP)} - {/*{["visiumst","visium"].includes(record.assay.toLowerCase()) && (Visium)}*/} - {["eqtl", "caqtl"].includes(record.assay.toLowerCase()) && (xQTL)} - {record.has_bw && (Peaks)} + {["scrnaseq", "snrnaseq", "visiumst", "merfish"].includes(record.assay.toLowerCase()) && (UMAP)} + {/*{["visiumst","visium"].includes(record.assay.toLowerCase()) && (Visium)}*/} + {["eqtl", "caqtl"].includes(record.assay.toLowerCase()) && (xQTL)} + {record.has_bw && (Peaks)} {deleteMode && ( diff --git a/frontend/src/pages/Datasets/LoadRemoteDatasetDialog.jsx b/frontend/src/pages/Datasets/LoadRemoteDatasetDialog.jsx new file mode 100644 index 0000000..92460f4 --- /dev/null +++ b/frontend/src/pages/Datasets/LoadRemoteDatasetDialog.jsx @@ -0,0 +1,223 @@ +import { useState } from "react"; +import { + Dialog, + DialogTitle, + DialogContent, + DialogActions, + Button, + TextField, + Box, + Typography, + Chip, + IconButton, + List, + ListItem, + ListItemText, + Divider, + Link, + Alert, +} from "@mui/material"; +import DeleteIcon from "@mui/icons-material/Delete"; +import PropTypes from "prop-types"; +import { toast } from "react-toastify"; + +import useRemoteDatasetStore, { + buildRemoteRecord, +} from "../../store/RemoteDatasetStore.js"; +import useDatatableStore from "../../store/DatatableStore.js"; +import { inspectDataset } from "../../api/qtl.js"; + +const normalizeUrl = (raw) => (raw || "").trim().replace(/\/+$/, ""); + +const LoadRemoteDatasetDialog = ({ open, onClose }) => { + const { remoteDatasets, addRemoteDataset, removeRemoteDataset } = + useRemoteDatasetStore(); + const { fetchDatasetList } = useDatatableStore(); + + const [url, setUrl] = useState(""); + const [name, setName] = useState(""); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(""); + + const reset = () => { + setUrl(""); + setName(""); + setError(""); + setLoading(false); + }; + + const handleClose = () => { + reset(); + onClose(); + }; + + const handleAdd = async () => { + setError(""); + const cleanUrl = normalizeUrl(url); + if (!/^https?:\/\//i.test(cleanUrl)) { + setError("Please enter a full http:// or https:// URL."); + return; + } + + setLoading(true); + try { + const info = await inspectDataset(cleanUrl); + if (!info || !info.reachable) { + setError( + info?.error || + "Could not reach a valid dataset at that URL. Check the URL, that the host is publicly reachable, and that it serves the dataset files (e.g. dataset_info.toml, celltypes/ or bigwig/).", + ); + setLoading(false); + return; + } + if (!info.has_qtl && !info.has_bw) { + setError( + "The URL is reachable but does not look like a QTL or signal/peaks dataset (no celltypes/ parquet and no bigwig/ tracks found).", + ); + setLoading(false); + return; + } + + const record = buildRemoteRecord(cleanUrl, name, info); + addRemoteDataset(record); + await fetchDatasetList(); + toast.success(`Added remote dataset "${record.dataset_name}".`); + reset(); + } catch (e) { + setError( + e?.response?.data?.detail || + "Failed to inspect the dataset URL. The server may have blocked it (private/loopback addresses are not allowed).", + ); + setLoading(false); + } + }; + + const handleRemove = async (datasetId) => { + removeRemoteDataset(datasetId); + await fetchDatasetList(); + toast.info("Removed remote dataset."); + }; + + return ( + + Load dataset from URL + + + Point to the dataset directory of a remotely hosted HTTP + server. The dataset is stored only in your browser and is + not visible to other users. + + + + setUrl(e.target.value)} + fullWidth + size="small" + autoFocus + /> + setName(e.target.value)} + fullWidth + size="small" + /> + {error && {error}} + + + {remoteDatasets.length > 0 && ( + <> + + + Your remote datasets + + + {remoteDatasets.map((ds) => ( + + handleRemove(ds.dataset_id) + } + > + + + } + > + + {ds.dataset_name} + + {ds.has_bw && ( + + )} + + } + secondary={ + + {ds.dataset_id} + + } + /> + + ))} + + + )} + + + + + + + ); +}; + +LoadRemoteDatasetDialog.propTypes = { + open: PropTypes.bool.isRequired, + onClose: PropTypes.func.isRequired, +}; + +export default LoadRemoteDatasetDialog; diff --git a/frontend/src/pages/Datasets/index.jsx b/frontend/src/pages/Datasets/index.jsx index d2c10f3..252be5a 100644 --- a/frontend/src/pages/Datasets/index.jsx +++ b/frontend/src/pages/Datasets/index.jsx @@ -3,6 +3,7 @@ import {Box, Typography, Divider, Button, Link} from "@mui/material"; import FilterPanel from "./FilterPanel"; import DatasetDisplay from "./DatasetDisplay.jsx"; +import LoadRemoteDatasetDialog from "./LoadRemoteDatasetDialog.jsx"; import "./DatasetPage.css"; import useDatatableStore from "../../store/DatatableStore.js"; @@ -123,6 +124,8 @@ const DatasetsPage = () => { setDeleteMode(!deleteMode); } + const [loadUrlOpen, setLoadUrlOpen] = useState(false); + return (
{/* Title Row */} @@ -141,6 +144,10 @@ const DatasetsPage = () => { {/* {deleteMode ? "Cancel Delete" : "- Delete Dataset"}*/} {/**/} + + @@ -148,6 +155,8 @@ const DatasetsPage = () => { + setLoadUrlOpen(false)}/> + {/* Main Content */} {/* Left Filter Panel */} diff --git a/frontend/src/store/DatatableStore.js b/frontend/src/store/DatatableStore.js index 22b567a..5acc61f 100644 --- a/frontend/src/store/DatatableStore.js +++ b/frontend/src/store/DatatableStore.js @@ -2,6 +2,7 @@ import {create} from "zustand"; import {toast} from "react-toastify"; import {getDatatable_get, getDatasetList, getSampletable_get} from "../api/api.js"; import {checkBWDataExists} from "./GenomicRegionStore.js"; +import useRemoteDatasetStore from "./RemoteDatasetStore.js"; const useDatatableStore = create((set) => ({ dataRecords: [], @@ -61,6 +62,7 @@ const useDatatableStore = create((set) => ({ }, fetchDatasetList: async () => { + const remoteDatasets = useRemoteDatasetStore.getState().remoteDatasets || []; try { const response = await getDatasetList(); // console.log(response); @@ -75,18 +77,21 @@ const useDatatableStore = create((set) => ({ }) ); - await set({datasetRecords: recordsWithBW, datasetFilters: data[1], datasetfetchStatus: "success"}); + await set({datasetRecords: [...recordsWithBW, ...remoteDatasets], datasetFilters: data[1], datasetfetchStatus: "success"}); // toast.success("Sample loaded successfully!"); } else { console.error("Error fetching data:", response.data); - await set({datasetRecords: [], datasetfetchStatus: "failed"}); + await set({datasetRecords: [...remoteDatasets], datasetfetchStatus: "failed"}); toast.error("Failed to fetch datasets."); } } catch (error) { console.error("Error fetching data:", error); - await set({datasetRecords: [], datasetfetchStatus: "error"}); - toast.info(error.response.data.detail); + // Use browser-local remote datasets if the server dataset table is unavailable. + await set({datasetRecords: [...remoteDatasets], datasetfetchStatus: "error"}); + if (error?.response?.data?.detail) { + toast.info(error.response.data.detail); + } } }, diff --git a/frontend/src/store/RemoteDatasetStore.js b/frontend/src/store/RemoteDatasetStore.js new file mode 100644 index 0000000..462c4d4 --- /dev/null +++ b/frontend/src/store/RemoteDatasetStore.js @@ -0,0 +1,64 @@ +import { create } from "zustand"; +import { persist } from "zustand/middleware"; + +export const buildRemoteRecord = (url, name, info = {}) => { + let assay = info.assay || ""; + const hasBw = !!info.has_bw; + const hasQtl = !!info.has_qtl; + + // Normalize the assay string so the view filters recognize it: + // - xQTL view lists datasets whose assay ends with "qtl" + // - Genomic-Region view lists datasets whose assay ends with RNAseq/ATACseq + if (hasQtl && !assay.toLowerCase().endsWith("qtl")) { + assay = info.is_caqtl ? "caQTL" : "eQTL"; + } + if (hasBw && !/(RNAseq|ATACseq)$/i.test(assay)) { + assay = + assay && assay.toLowerCase().endsWith("qtl") ? assay : "scATACseq"; + } + + return { + dataset_id: url, + dataset_name: name || info.dataset_name || url, + PI_full_name: (info.info && info.info.PI_full_name) || "—", + first_contributor: (info.info && info.info.first_contributor) || "—", + n_samples: (info.info && info.info.n_samples) || 0, + brain_region: (info.info && info.info.brain_region) || "", + brain_super_region: (info.info && info.info.brain_super_region) || "", + disease: (info.info && info.info.disease) || "", + organism: (info.info && info.info.organism) || "", + tissue: (info.info && info.info.tissue) || "", + assay: assay, + sample_sheet: "None", + has_bw: hasBw, + is_remote: true, + }; +}; + +const useRemoteDatasetStore = create( + persist( + (set) => ({ + remoteDatasets: [], + + addRemoteDataset: (record) => + set((state) => ({ + remoteDatasets: [ + ...state.remoteDatasets.filter( + (r) => r.dataset_id !== record.dataset_id, + ), + record, + ], + })), + + removeRemoteDataset: (datasetId) => + set((state) => ({ + remoteDatasets: state.remoteDatasets.filter( + (r) => r.dataset_id !== datasetId, + ), + })), + }), + { name: "vizit-remote-datasets" }, + ), +); + +export default useRemoteDatasetStore; From 11860003d92e551af3b8e002616271af5207de7d Mon Sep 17 00:00:00 2001 From: Rehpotsirhc <86068565+Rehpotsirhc-z@users.noreply.github.com> Date: Mon, 8 Jun 2026 15:47:16 -0400 Subject: [PATCH 2/6] Use display name for remote backend --- frontend/src/pages/Datasets/DatasetDisplay.jsx | 2 +- frontend/src/pages/GenomicRegionView/index.jsx | 9 ++++++++- frontend/src/pages/XQTLView/index.jsx | 9 ++++++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/frontend/src/pages/Datasets/DatasetDisplay.jsx b/frontend/src/pages/Datasets/DatasetDisplay.jsx index 8165199..f75a7fc 100644 --- a/frontend/src/pages/Datasets/DatasetDisplay.jsx +++ b/frontend/src/pages/Datasets/DatasetDisplay.jsx @@ -208,7 +208,7 @@ const DatasetDisplay = ({dataRecords, deleteMode}) => { {displayedData.map((record) => ( - {record.sample_sheet === "None" || record.sample_sheet === null || record.sample_sheet.trim() === "" ? record.dataset_id : + {record.is_remote || record.sample_sheet === "None" || record.sample_sheet === null || record.sample_sheet.trim() === "" ? (record.dataset_name || record.dataset_id) : {record.dataset_id}} {record.PI_full_name} diff --git a/frontend/src/pages/GenomicRegionView/index.jsx b/frontend/src/pages/GenomicRegionView/index.jsx index b93ea18..3e02cc2 100644 --- a/frontend/src/pages/GenomicRegionView/index.jsx +++ b/frontend/src/pages/GenomicRegionView/index.jsx @@ -86,6 +86,12 @@ function GenomicRegionView() { .filter((d) => /[A-Za-z]*?(RNAseq|ATACseq)$/i.test(d.assay) && d.has_bw) .map((d) => d.dataset_id); + // Map dataset_id to display name + const datasetLabel = (id) => { + const ds = datasetRecords.find((d) => d.dataset_id === id); + return (ds && ds.dataset_name) || id || ""; + }; + const [datasetId, setDatasetId] = useState(urlDataset); const [datasetSearchText, setDatasetSearchText] = useState(""); @@ -786,6 +792,7 @@ function GenomicRegionView() { options={datasetOptions} value={datasetId ?? null} onChange={handleDatasetChange} + getOptionLabel={(option) => datasetLabel(option)} inputValue={datasetSearchText} onInputChange={(event, newInputValue) => setDatasetSearchText(newInputValue) @@ -802,7 +809,7 @@ function GenomicRegionView() { const { key, ...rest } = props; return (
  • - {option} + {datasetLabel(option)}
  • // // {option} diff --git a/frontend/src/pages/XQTLView/index.jsx b/frontend/src/pages/XQTLView/index.jsx index 6198e44..5fb92b6 100644 --- a/frontend/src/pages/XQTLView/index.jsx +++ b/frontend/src/pages/XQTLView/index.jsx @@ -87,6 +87,12 @@ function XQTLView() { .filter((d) => d.assay.toLowerCase().endsWith("qtl")) .map((d) => d.dataset_id); + // Map dataset_id to display name + const datasetLabel = (id) => { + const ds = datasetRecords.find((d) => d.dataset_id === id); + return (ds && ds.dataset_name) || id || ""; + }; + const [datasetId, setDatasetId] = useState(urlDataset); const [datasetSearchText, setDatasetSearchText] = useState(""); @@ -551,6 +557,7 @@ function XQTLView() { options={datasetOptions} value={datasetId ?? null} onChange={handleDatasetChange} + getOptionLabel={(option) => datasetLabel(option)} inputValue={datasetSearchText} onInputChange={(event, newInputValue) => setDatasetSearchText(newInputValue) @@ -567,7 +574,7 @@ function XQTLView() { const { key, ...rest } = props; return (
  • - {option} + {datasetLabel(option)}
  • // // {option} From 9505cd5d77c6aef94ed518e0d8241d9abe3c5f4f Mon Sep 17 00:00:00 2001 From: Rehpotsirhc <86068565+Rehpotsirhc-z@users.noreply.github.com> Date: Tue, 9 Jun 2026 12:08:55 -0400 Subject: [PATCH 3/6] Clear cache on remote dataset inspect --- backend/funcs/remote_io.py | 6 +++++- backend/routes/qtl_routes.py | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/backend/funcs/remote_io.py b/backend/funcs/remote_io.py index 0b32c6b..67c4dca 100644 --- a/backend/funcs/remote_io.py +++ b/backend/funcs/remote_io.py @@ -135,7 +135,11 @@ def _remote_exists(url: str) -> bool: resp.close() -@lru_cache(maxsize=64) +def clear_remote_cache() -> None: + _fetch_bytes_cached.cache_clear() + + +@lru_cache(maxsize=512) def _fetch_bytes_cached(url: str) -> bytes: try: resp = _request("GET", url, stream=True) diff --git a/backend/routes/qtl_routes.py b/backend/routes/qtl_routes.py index c377ffd..36a5e27 100644 --- a/backend/routes/qtl_routes.py +++ b/backend/routes/qtl_routes.py @@ -3,6 +3,7 @@ # from backend.funcs.get_data import * +from backend.funcs.remote_io import is_remote, clear_remote_cache from backend.funcs.get_data import ( get_qtl_gene_list, get_qtl_snp_list, @@ -33,6 +34,8 @@ async def read_root(): @router.get("/inspectdataset") async def inspectdataset(request: Request): dataset_id = request.query_params.get("dataset") + if is_remote(dataset_id): + clear_remote_cache() return inspect_dataset(dataset_id) From dc90b1051313f30180c2a87e63ff3837ae5a0717 Mon Sep 17 00:00:00 2001 From: Rehpotsirhc <86068565+Rehpotsirhc-z@users.noreply.github.com> Date: Tue, 9 Jun 2026 12:45:02 -0400 Subject: [PATCH 4/6] Fetch dataset_name on shared link --- .../src/pages/GenomicRegionView/index.jsx | 17 ++++++++++++ frontend/src/pages/XQTLView/index.jsx | 17 ++++++++++++ frontend/src/store/RemoteDatasetStore.js | 27 ++++++++++++++++++- 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/frontend/src/pages/GenomicRegionView/index.jsx b/frontend/src/pages/GenomicRegionView/index.jsx index 3e02cc2..516ffce 100644 --- a/frontend/src/pages/GenomicRegionView/index.jsx +++ b/frontend/src/pages/GenomicRegionView/index.jsx @@ -33,6 +33,7 @@ import "./GenomicRegionView.css"; import useDataStore from "../../store/DatatableStore.js"; import useSignalStore from "../../store/GenomicRegionStore.js"; +import useRemoteDatasetStore from "../../store/RemoteDatasetStore.js"; import RegionViewPlotlyPlot from "./RegionViewPlotlyPlot.jsx"; @@ -77,11 +78,27 @@ function GenomicRegionView() { const urlRegion = queryParams.get("region") ?? ""; const { datasetRecords, fetchDatasetList } = useDataStore(); + const { ensureRemoteDataset } = useRemoteDatasetStore(); useEffect(() => { fetchDatasetList(); }, []); + // When a remote dataset is referenced by URL but not registered locally + // (e.g. when pasting in a shared link), register it so its display name is + // fetched from the dataset_info.toml + useEffect(() => { + if ( + urlDataset && + /^https?:\/\//i.test(urlDataset) && + !datasetRecords.some((d) => d.dataset_id === urlDataset) + ) { + ensureRemoteDataset(urlDataset).then((added) => { + if (added) fetchDatasetList(); + }); + } + }, [urlDataset, datasetRecords]); + const datasetOptions = datasetRecords .filter((d) => /[A-Za-z]*?(RNAseq|ATACseq)$/i.test(d.assay) && d.has_bw) .map((d) => d.dataset_id); diff --git a/frontend/src/pages/XQTLView/index.jsx b/frontend/src/pages/XQTLView/index.jsx index 5fb92b6..fa9a146 100644 --- a/frontend/src/pages/XQTLView/index.jsx +++ b/frontend/src/pages/XQTLView/index.jsx @@ -33,6 +33,7 @@ import "./XQTLView.css"; import useDataStore from "../../store/DatatableStore.js"; import useQtlStore from "../../store/QtlStore.js"; +import useRemoteDatasetStore from "../../store/RemoteDatasetStore.js"; import GeneViewPlotlyPlot from "./GeneViewPlotlyPlot.jsx"; import SNPViewPlotlyPlot from "./SNPViewPlotlyPlot.jsx"; @@ -79,10 +80,26 @@ function XQTLView() { const urlDataset = queryParams.get("dataset") ?? ""; const { datasetRecords, fetchDatasetList } = useDataStore(); + const { ensureRemoteDataset } = useRemoteDatasetStore(); useEffect(() => { fetchDatasetList(); }, []); + // When a remote dataset is referenced by URL but not registered locally + // (e.g. when pasting in a shared link), register it so its display name is + // fetched from the dataset_info.toml + useEffect(() => { + if ( + urlDataset && + /^https?:\/\//i.test(urlDataset) && + !datasetRecords.some((d) => d.dataset_id === urlDataset) + ) { + ensureRemoteDataset(urlDataset).then((added) => { + if (added) fetchDatasetList(); + }); + } + }, [urlDataset, datasetRecords]); + const datasetOptions = datasetRecords .filter((d) => d.assay.toLowerCase().endsWith("qtl")) .map((d) => d.dataset_id); diff --git a/frontend/src/store/RemoteDatasetStore.js b/frontend/src/store/RemoteDatasetStore.js index 462c4d4..0f3f76b 100644 --- a/frontend/src/store/RemoteDatasetStore.js +++ b/frontend/src/store/RemoteDatasetStore.js @@ -1,6 +1,8 @@ import { create } from "zustand"; import { persist } from "zustand/middleware"; +import { inspectDataset } from "../api/qtl.js"; + export const buildRemoteRecord = (url, name, info = {}) => { let assay = info.assay || ""; const hasBw = !!info.has_bw; @@ -37,7 +39,7 @@ export const buildRemoteRecord = (url, name, info = {}) => { const useRemoteDatasetStore = create( persist( - (set) => ({ + (set, get) => ({ remoteDatasets: [], addRemoteDataset: (record) => @@ -50,6 +52,29 @@ const useRemoteDatasetStore = create( ], })), + ensureRemoteDataset: async (url) => { + if (!url || !/^https?:\/\//i.test(url)) return false; + if (get().remoteDatasets.some((r) => r.dataset_id === url)) { + return false; + } + try { + const info = await inspectDataset(url); + if (!info || !info.reachable) return false; + const record = buildRemoteRecord(url, "", info); + set((state) => ({ + remoteDatasets: [ + ...state.remoteDatasets.filter( + (r) => r.dataset_id !== url, + ), + record, + ], + })); + return true; + } catch { + return false; + } + }, + removeRemoteDataset: (datasetId) => set((state) => ({ remoteDatasets: state.remoteDatasets.filter( From 9e7324e501fb4b6e372a6fb58bcf766d3c26a57e Mon Sep 17 00:00:00 2001 From: Rehpotsirhc <86068565+Rehpotsirhc-z@users.noreply.github.com> Date: Tue, 9 Jun 2026 13:53:34 -0400 Subject: [PATCH 5/6] Fix dataset list for remote entries --- frontend/src/store/DatatableStore.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/frontend/src/store/DatatableStore.js b/frontend/src/store/DatatableStore.js index 5acc61f..c3f2523 100644 --- a/frontend/src/store/DatatableStore.js +++ b/frontend/src/store/DatatableStore.js @@ -97,4 +97,11 @@ const useDatatableStore = create((set) => ({ })); +// When remote datasets change, refresh the merged list so all views pick up the +// updated remote datasets. +useRemoteDatasetStore.subscribe( + (state) => state.remoteDatasets, + () => useDatatableStore.getState().fetchDatasetList(), +); + export default useDatatableStore; From cd319575351ac2774025f8fdb2b3ea0208ceaf95 Mon Sep 17 00:00:00 2001 From: Rehpotsirhc <86068565+Rehpotsirhc-z@users.noreply.github.com> Date: Tue, 9 Jun 2026 14:16:30 -0400 Subject: [PATCH 6/6] Add remote capabilities endpoint Don't allow loading from URL if server does not support it --- backend/funcs/remote_capabilities.py | 151 ++++++++++++++++++++++++ backend/main.py | 45 ++++++- frontend/src/api/api.js | 10 ++ frontend/src/pages/Datasets/index.jsx | 17 ++- frontend/src/store/ServerConfigStore.js | 29 +++++ 5 files changed, 247 insertions(+), 5 deletions(-) create mode 100644 backend/funcs/remote_capabilities.py create mode 100644 frontend/src/store/ServerConfigStore.js diff --git a/backend/funcs/remote_capabilities.py b/backend/funcs/remote_capabilities.py new file mode 100644 index 0000000..ca46465 --- /dev/null +++ b/backend/funcs/remote_capabilities.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +import importlib +import logging +from dataclasses import dataclass, field +from typing import Callable, List, Optional + +logger = logging.getLogger("vizit.remote_capabilities") + + +@dataclass +class CapabilityResult: + name: str # short id + required_for: str # description of the feature + ok: Optional[bool] # True = available, False = missing, None = could not determine + detail: str # description of the check and its result + fix_hint: str = "" # how to install + + +# --------------------------------------------------------------------------- # +# Registry +# --------------------------------------------------------------------------- # +CapabilityCheck = Callable[[], CapabilityResult] +_CHECKS: List[CapabilityCheck] = [] + + +def register_capability_check(fn: CapabilityCheck) -> CapabilityCheck: + _CHECKS.append(fn) + return fn + + +def run_capability_checks() -> List[CapabilityResult]: + results: List[CapabilityResult] = [] + for fn in _CHECKS: + try: + results.append(fn()) + except Exception as exc: + results.append( + CapabilityResult( + name=getattr(fn, "__name__", "unknown"), + required_for="unknown", + ok=None, + detail=f"capability check raised {type(exc).__name__}: {exc}", + ) + ) + return results + + +# --------------------------------------------------------------------------- # +# Probes +# --------------------------------------------------------------------------- # +def _module_available(modname: str) -> bool: + try: + importlib.import_module(modname) + return True + except Exception: + return False + + +@register_capability_check +def check_polars_http() -> CapabilityResult: + have_fsspec = _module_available("fsspec") + have_aiohttp = _module_available("aiohttp") + ok = have_fsspec and have_aiohttp + if ok: + detail = "fsspec + aiohttp present; polars can read parquet/CSV over http(s)." + fix = "" + else: + missing = [ + m + for m, present in (("fsspec", have_fsspec), ("aiohttp", have_aiohttp)) + if not present + ] + detail = ( + f"missing {', '.join(missing)}; polars cannot read parquet/CSV from " + "remote URLs (gene/SNP locations, QTL parquet, GWAS tables)." + ) + fix = "pip install fsspec aiohttp" + return CapabilityResult( + name="polars-http", + required_for="remote QTL parquet / gene & SNP locations / GWAS tables", + ok=ok, + detail=detail, + fix_hint=fix, + ) + + +@register_capability_check +def check_pybigwig_remote() -> CapabilityResult: + try: + import pyBigWig + except Exception as exc: + return CapabilityResult( + name="pybigwig-remote", + required_for="remote BigWig signal tracks", + ok=None, + detail=f"pyBigWig import failed: {type(exc).__name__}: {exc}", + fix_hint="pip install pyBigWig", + ) + + ok = bool(getattr(pyBigWig, "remote", 0)) + if ok: + detail = ( + "pyBigWig was built with libcurl; remote .bw/.bigWig URLs are supported." + ) + fix = "" + else: + detail = ( + "pyBigWig was built WITHOUT libcurl; " + "remote BigWig signal tracks will fail to open." + ) + fix = ( + "Install a curl-enabled build, e.g. install libcurl " + "and then run `pip install --no-binary pyBigWig pyBigWig`" + ) + return CapabilityResult( + name="pybigwig-remote", + required_for="remote BigWig signal tracks", + ok=ok, + detail=detail, + fix_hint=fix, + ) + + +# --------------------------------------------------------------------------- # +# Startup +# --------------------------------------------------------------------------- # +def warn_on_missing_remote_capabilities() -> List[CapabilityResult]: + results = run_capability_checks() + missing = [r for r in results if r.ok is False] + unknown = [r for r in results if r.ok is None] + + if not missing and not unknown: + logger.info( + "Remote datasets enabled; all %d optional capabilities available.", + len(results), + ) + return results + + logger.warning( + "Remote datasets are ENABLED but %d capability(ies) are unavailable. " + "Affected features will error for remote datasets:", + len(missing) + len(unknown), + ) + for r in missing + unknown: + state = "MISSING" if r.ok is False else "UNKNOWN" + logger.warning(" [%s] %s — required for: %s", state, r.name, r.required_for) + logger.warning(" %s", r.detail) + if r.fix_hint: + logger.warning(" fix: %s", r.fix_hint) + return results diff --git a/backend/main.py b/backend/main.py index e413502..97a367b 100644 --- a/backend/main.py +++ b/backend/main.py @@ -4,18 +4,38 @@ from starlette.requests import Request from backend.db import create_db_and_tables -from backend.routes import db_routes, api_routes, visium_routes, qtl_routes, dm_routes,signal_routes +from backend.routes import ( + db_routes, + api_routes, + visium_routes, + qtl_routes, + dm_routes, + signal_routes, +) from backend.funcs.remote_io import RemoteDatasetError +from backend.funcs.remote_capabilities import ( + warn_on_missing_remote_capabilities, + run_capability_checks, +) from backend.settings import settings app = FastAPI(debug=settings.debug) +def check_remote_capabilities_on_startup(): + # Only relevant remote datasets is enabled + if settings.allow_remote_datasets: + warn_on_missing_remote_capabilities() + + @app.exception_handler(RemoteDatasetError) async def remote_dataset_error_handler(request: Request, exc: RemoteDatasetError): # A remote (client-supplied) dataset URL was disallowed or unreachable. - return JSONResponse(status_code=400, content={"detail": f"Remote dataset error: {exc}"}) + return JSONResponse( + status_code=400, content={"detail": f"Remote dataset error: {exc}"} + ) + app.add_middleware( CORSMiddleware, @@ -28,11 +48,31 @@ async def remote_dataset_error_handler(request: Request, exc: RemoteDatasetError ) app.add_event_handler("startup", create_db_and_tables) +app.add_event_handler("startup", check_remote_capabilities_on_startup) + @app.get("/") async def root(): return "Hello, Welcome to VizIt!" + +@app.get("/serverconfig") +async def server_config(): + caps = run_capability_checks() if settings.allow_remote_datasets else [] + return { + "allow_remote_datasets": settings.allow_remote_datasets, + "remote_capabilities": [ + { + "name": c.name, + "required_for": c.required_for, + "ok": c.ok, + "detail": c.detail, + } + for c in caps + ], + } + + app.include_router(db_routes.router, prefix="/db") app.include_router(api_routes.router, prefix="/api") app.include_router(visium_routes.router, prefix="/visium") @@ -42,6 +82,7 @@ async def root(): if __name__ == "__main__": import uvicorn + print("Starting FastAPI server on port", settings.uvicorn_port) uvicorn.run(app, host=settings.uvicorn_host, port=settings.uvicorn_port) diff --git a/frontend/src/api/api.js b/frontend/src/api/api.js index 9d274c6..f1066ad 100644 --- a/frontend/src/api/api.js +++ b/frontend/src/api/api.js @@ -3,6 +3,16 @@ import axios from "axios"; const BASE_URL = import.meta.env.VITE_BACKEND_URL; const API_URL = `${BASE_URL}/api`; +export const getServerConfig = async () => { + try { + const response = await axios.get(`${BASE_URL}/serverconfig`); + return response.data; + } catch (error) { + console.error("Error getServerConfig:", error); + throw error; + } +} + export const getHomeData = async () => { try { const response = await axios.get(`${API_URL}/gethomedata`); diff --git a/frontend/src/pages/Datasets/index.jsx b/frontend/src/pages/Datasets/index.jsx index 252be5a..587a015 100644 --- a/frontend/src/pages/Datasets/index.jsx +++ b/frontend/src/pages/Datasets/index.jsx @@ -6,11 +6,13 @@ import DatasetDisplay from "./DatasetDisplay.jsx"; import LoadRemoteDatasetDialog from "./LoadRemoteDatasetDialog.jsx"; import "./DatasetPage.css"; import useDatatableStore from "../../store/DatatableStore.js"; +import useServerConfigStore from "../../store/ServerConfigStore.js"; import {useSearchParams} from "react-router-dom"; const DatasetsPage = () => { const {datasetRecords, fetchDatasetList} = useDatatableStore(); + const {allowRemoteDatasets, fetchServerConfig} = useServerConfigStore(); const [searchParams, setSearchParams] = useSearchParams(); // Initialize filters from URL params @@ -27,6 +29,13 @@ const DatasetsPage = () => { fetchDatasetList() }, [fetchDatasetList]); + useEffect(() => { + fetchServerConfig() + }, [fetchServerConfig]); + + // null keeps button visible for backward compatibility + const remoteEnabled = allowRemoteDatasets !== false; + // Update URL when filters change useEffect(() => { const newSearchParams = new URLSearchParams() @@ -144,9 +153,11 @@ const DatasetsPage = () => { {/* {deleteMode ? "Cancel Delete" : "- Delete Dataset"}*/} {/**/} - + {remoteEnabled && ( + + )}