diff --git a/backend/db_utils/refresh_db.py b/backend/db_utils/refresh_db.py new file mode 100644 index 0000000..654ac94 --- /dev/null +++ b/backend/db_utils/refresh_db.py @@ -0,0 +1,180 @@ +import os +import shutil +import sys + +import toml +from sqlmodel import Session + +from backend.db import engine, create_db_and_tables +from backend.db_utils.crud import insert_study, insert_dataset, import_sample_sheet +from backend.models import Study, Dataset + +DATASETS_DIR = "backend/datasets" +SAMPLESHEETS_DIR = "backend/SampleSheets" + + +def check_toml_file(toml_data): + if "datasetfile" not in toml_data: + return {"message": "Error: Missing [datasetfile] config.", "success": False} + else: + required_keys = ["datatype"] + for key in required_keys: + if ( + key not in toml_data["datasetfile"] + or toml_data["datasetfile"][key].strip() == "" + ): + return { + "message": f"Error: Missing [datasetfile - {key}] config.", + "success": False, + } + + if "dataset" not in toml_data: + return {"message": "Error: Missing [dataset] config.", "success": False} + else: + required_keys = [ + "dataset_name", + "n_samples", + "brain_super_region", + "brain_region", + "organism", + "tissue", + "disease", + ] + for key in required_keys: + if ( + key not in toml_data["dataset"] + or str(toml_data["dataset"][key]).strip() == "" + ): + return { + "message": f"Error: Missing [dataset - {key}] config.", + "success": False, + } + + if "study" not in toml_data: + return {"message": "Error: Missing [study] config.", "success": False} + else: + required_keys = ["study_name"] + for key in required_keys: + if key not in toml_data["study"] or toml_data["study"][key].strip() == "": + return { + "message": f"Error: Missing [study - {key}] config.", + "success": False, + } + + if "protocol" not in toml_data: + return {"message": "Error: Missing [protocol] config.", "success": False} + else: + required_keys = ["protocol_id"] + for key in required_keys: + if ( + key not in toml_data["protocol"] + or toml_data["protocol"][key].strip() == "" + ): + return { + "message": f"Error: Missing [protocol - {key}] config.", + "success": False, + } + + return {"message": "Config file is valid", "success": True} + + +def refresh_dataset(dataset_i, session: Session): + """Import a single dataset folder. Returns the check_toml_file-style result.""" + dataset_path = f"{DATASETS_DIR}/{dataset_i}" + ## load dataset info + dataset_info_file = f"{dataset_path}/dataset_info.toml" + if not os.path.exists(dataset_info_file): + return {"message": "Skipped: no dataset_info.toml.", "success": None} + + with open(dataset_info_file, "r") as f: + dataset_info = toml.load(f) + + ## check if dataset configuration is valid + check_result = check_toml_file(dataset_info) + if not check_result["success"]: + return check_result + + print("=======insert info into database==========") + study_dict = dataset_info["study"] + study_dict["study_id"] = study_dict["study_name"] + study = Study(**study_dict) + + dataset_dict = dataset_info["dataset"] + dataset_dict["dataset_id"] = dataset_dict["dataset_name"] + dataset_dict["assay"] = dataset_info["datasetfile"]["datatype"] + dataset_dict["study_id"] = study_dict["study_id"] + dataset_dict["dataset_file"] = dataset_info["datasetfile"]["file"] + dataset = Dataset(**dataset_dict) + + insert_study(study, session) + insert_dataset(dataset, session) + + sample_sheet = dataset_dict.get("sample_sheet") + if sample_sheet not in (None, "None", ""): + sample_sheet_path = f"{SAMPLESHEETS_DIR}/{sample_sheet}" + shutil.copyfile(sample_sheet_path, f"{dataset_path}/{sample_sheet}") + import_sample_sheet(sample_sheet_path, session) + + return {"message": f"Imported '{dataset_i}'.", "success": True} + + +def refresh_database(session: Session): + """Import every dataset folder. A dataset with an invalid or unreadable + dataset_info.toml is reported and skipped, the remaining ones still run.""" + if not os.path.isdir(DATASETS_DIR): + return { + "message": f"Error: '{DATASETS_DIR}' not found. Run this from the repository root.", + "success": False, + "imported": 0, + "skipped": 0, + "errors": [], + } + + imported = 0 + errors = [] + ## loop through all datasets + for dataset_i in sorted(os.listdir(DATASETS_DIR)): + if not os.path.isdir(f"{DATASETS_DIR}/{dataset_i}"): + continue + try: + result = refresh_dataset(dataset_i, session) + except Exception as e: + errors.append(f"{dataset_i}: {e}") + continue + + if result["success"]: + imported += 1 + elif result["success"] is False: + errors.append(f"{dataset_i}: {result['message']}") + + if errors: + return { + "message": f"Database refreshed with problems: {imported} imported, {len(errors)} skipped.", + "success": False, + "imported": imported, + "skipped": len(errors), + "errors": errors, + } + + return { + "message": f"Database refreshed successfully: {imported} dataset(s).", + "success": True, + "imported": imported, + "skipped": 0, + "errors": [], + } + + +def main(): + create_db_and_tables() + with Session(engine) as session: + result = refresh_database(session) + + for error in result["errors"]: + print(f"[SKIPPED] {error}", file=sys.stderr) + print(result["message"]) + sys.exit(0 if result["success"] else 1) + + +if __name__ == "__main__": + main() diff --git a/backend/funcs/get_data.py b/backend/funcs/get_data.py index 34ea6e8..cc55437 100644 --- a/backend/funcs/get_data.py +++ b/backend/funcs/get_data.py @@ -9,6 +9,17 @@ import re from functools import lru_cache +from backend.funcs.remote_io import ( + is_remote, + ds_locator, + ds_exists, + ds_read_json, + ds_read_text, + ds_read_toml, + ds_text_stream, + ds_bytes_stream, +) + def safe_filename(name): return re.sub(r"[^a-zA-Z0-9_\-]", "_", name) @@ -64,22 +75,17 @@ def get_gene_location(dataset, gene): return f"Error: Could not parse positions from gene {gene}: {str(e)}" # Case 2: eQTL dataset - need to look up in JSON file - genes_file = os.path.join( - "backend", "datasets", dataset, "gene_jsons", - get_gene_file_name(safe_filename(gene)) + ".json" - ) + gene_file = get_gene_file_name(safe_filename(gene)) + ".json" + data = ds_read_json(dataset, "gene_jsons", gene_file) - if not os.path.exists(genes_file): - print(genes_file + " not found") + if data is None: + print(f"gene_jsons/{gene_file} not found for {dataset}") return "Error: Gene list file not found for the specified dataset." - - with open(genes_file, "r") as f: - data = json.load(f) - - if isinstance(data, dict) and gene in data: - gene_x = data[gene] - else: - gene_x = data + + if isinstance(data, dict) and gene in data: + gene_x = data[gene] + else: + gene_x = data if not gene_x: return f"Error: Gene {gene} not found in dataset." @@ -105,19 +111,10 @@ def get_snp_location(dataset, snp): if dataset == "all": return "Error: Dataset is not specified." - def load_snp_from_file(file_path): - if os.path.exists(file_path): - with open(file_path, "r") as f: - data = json.load(f) - return data.get(snp, None) - return None - try: group = get_snp_group(snp) - snps_file = os.path.join( - "backend", "datasets", dataset, "snp_jsons_merged", group + ".json" - ) - snp_data = load_snp_from_file(snps_file) + group_map = ds_read_json(dataset, "snp_jsons_merged", group + ".json") + snp_data = group_map.get(snp) if isinstance(group_map, dict) else None if snp_data: return { "chromosome": snp_data["chromosome"], @@ -128,12 +125,14 @@ def load_snp_from_file(file_path): pass default_file = os.path.join("backend", "datasets", "snp_locations.json") - snp_data = load_snp_from_file(default_file) - if snp_data: - return { - "chromosome": snp_data["chromosome"], - "position": snp_data["position"], - } + if os.path.exists(default_file): + with open(default_file, "r") as f: + snp_data = json.load(f).get(snp, None) + if snp_data: + return { + "chromosome": snp_data["chromosome"], + "position": snp_data["position"], + } return None @@ -142,13 +141,11 @@ def get_gene_locations_in_chromosome(dataset, chromosome, start, end): if dataset == "all": return "Error: Dataset is not specified." else: - chromosome_file = os.path.join( - "backend", "datasets", dataset, "gene_locations", chromosome + ".parquet" - ) + rel = ("gene_locations", chromosome + ".parquet") # significant_genes_list = get_qtl_gene_list(dataset) - if os.path.exists(chromosome_file): - df = pl.read_parquet(chromosome_file).filter( + if ds_exists(dataset, *rel): + df = pl.read_parquet(ds_locator(dataset, *rel)).filter( (pl.col("position_end") >= start) & (pl.col("position_start") <= end) # & pl.col("gene_id").is_in(significant_genes_list) @@ -161,7 +158,7 @@ def get_gene_locations_in_chromosome(dataset, chromosome, start, end): else: return f"No genes found in the region." else: - print(chromosome_file + " not found") + print(f"gene_locations/{chromosome}.parquet not found for {dataset}") return "Error: Chromosome file not found for the specified dataset." @@ -169,13 +166,11 @@ def get_snp_locations_in_chromosome(dataset, chromosome, start, end): if dataset == "all": return "Error: Dataset is not specified." else: - chromosome_file = os.path.join( - "backend", "datasets", dataset, "snp_locations", chromosome + ".parquet" - ) + rel = ("snp_locations", chromosome + ".parquet") significant_snps_list = get_qtl_snp_list(dataset) - if os.path.exists(chromosome_file): - df = pl.read_parquet(chromosome_file).filter( + if ds_exists(dataset, *rel): + df = pl.read_parquet(ds_locator(dataset, *rel)).filter( (pl.col("position") >= start) & (pl.col("position") <= end) & pl.col("snp_id").is_in(significant_snps_list) @@ -186,31 +181,28 @@ def get_snp_locations_in_chromosome(dataset, chromosome, start, end): else: return f"No SNPs found in the region." else: - print(chromosome_file + " not found") + print(f"snp_locations/{chromosome}.parquet not found for {dataset}") return "Error: Chromosome file not found for the specified dataset." def get_gwas_datasets(dataset): - config_path = os.path.join("backend", "datasets", dataset, "gwas", "gwas_datasets.toml") - if not os.path.exists(config_path): - return [] - with open(config_path, "r") as f: - config = toml.load(f) + config = ds_read_toml(dataset, "gwas", "gwas_datasets.toml") + if config is None: + return [] return config.get("datasets", []) def get_gwas_in_chromosome(qtl_dataset, gwas_dataset, chromosome, start, end): - base_dir = os.path.join("backend", "datasets", qtl_dataset, "gwas", gwas_dataset) - file_path = os.path.join(base_dir, f"{chromosome}.tsv") - if not os.path.exists(file_path): - # Return empty - return { - "snp_id": [], - "position": [], - "beta_value": [], - "p_value": [], - } - df = pl.read_csv(file_path, separator="\t").filter( + empty = { + "snp_id": [], + "position": [], + "beta_value": [], + "p_value": [], + } + tsv_stream = ds_bytes_stream(qtl_dataset, "gwas", gwas_dataset, f"{chromosome}.tsv") + if tsv_stream is None: + return empty + df = pl.read_csv(tsv_stream, separator="\t").filter( (pl.col("position") >= start) & (pl.col("position") <= end) ) if df.is_empty(): @@ -234,16 +226,10 @@ def get_gene_chromosome(dataset, gene): chromosome = gene.split("-")[0] return chromosome - genes_file = os.path.join( - "backend", "datasets", dataset, "gene_jsons", - get_gene_file_name(safe_filename(gene)) + ".json" - ) - - if os.path.exists(genes_file): - with open(genes_file, "r") as f: - data = json.load(f) - gene_x = data + gene_file = get_gene_file_name(safe_filename(gene)) + ".json" + gene_x = ds_read_json(dataset, "gene_jsons", gene_file) + if gene_x is not None: if gene_x: chromosome = gene_x.get("chromosome") if chromosome: @@ -253,7 +239,7 @@ def get_gene_chromosome(dataset, gene): else: return f"Error: Gene {gene} not found in {dataset} dataset." else: - print(genes_file + " not found") + print(f"gene_jsons/{gene_file} not found for {dataset}") return f"Error: Gene list file not found for the specified dataset {dataset}." @@ -261,32 +247,28 @@ def get_snp_chromosome(dataset, snp): if dataset == "all": return "Error: Dataset is not specified." else: - snps_file = os.path.join( - "backend", "datasets", dataset, "snp_jsons_merged", get_snp_group(snp) + ".json" - ) + snps_file = ("snp_jsons_merged", get_snp_group(snp) + ".json") - if os.path.exists(snps_file): - with open(snps_file, "r") as f: - data = json.load(f) - snp = data.get(snp, None) + data = ds_read_json(dataset, *snps_file) + if data is not None: + snp = data.get(snp, None) if data and snp: return snp["chromosome"] else: return f"Error: SNP {snp} not found in {dataset} dataset." else: - print(snps_file + " not found") + print(f"{'/'.join(snps_file)} not found for {dataset}") return "Error: SNP list file not found for the specified dataset." def get_gene_list(dataset, query_str="AB"): if dataset == "all": genes_file = os.path.join("backend", "datasets", "gene_list.json") + data = json.load(open(genes_file)) if os.path.exists(genes_file) else None else: - genes_file = os.path.join("backend", "datasets", dataset, "gene_list.json") + data = ds_read_json(dataset, "gene_list.json") - if os.path.exists(genes_file): - with open(genes_file, "r") as f: - data = json.load(f) + if data is not None: if query_str == "all": return data elif query_str == "default": @@ -294,22 +276,16 @@ def get_gene_list(dataset, query_str="AB"): else: return [gene for gene in data if gene.lower().startswith(query_str.lower())] else: - print(genes_file + " not found") + print(f"gene_list.json not found for {dataset}") return "Error: Gene list file not found" def get_qtl_gene_list(dataset, query_str="all"): if dataset == "all": return "Error: Gene dataset not specified." - else: - genes_file = os.path.join( - "backend", "datasets", dataset, "gene_list.json" - ) - - if os.path.exists(genes_file): - with open(genes_file, "r") as f: - data = json.load(f) + data = ds_read_json(dataset, "gene_list.json") + if data is not None: data = _normalize_gene_list(data) if query_str == "all": @@ -329,72 +305,62 @@ def get_qtl_gene_list(dataset, query_str="all"): "name": [g["name"] for g in filtered], } else: - print(genes_file + " not found") + print(f"gene_list.json not found for {dataset}") return "Error: Gene list file not found" def get_qtl_snp_list(dataset, query_str="all"): if dataset == "all": return "Error: SNP dataset not specified." - else: - snps_file = os.path.join("backend", "datasets", dataset, "snp_list.json") - if os.path.exists(snps_file): - with open(snps_file, "r") as f: - data = json.load(f) + data = ds_read_json(dataset, "snp_list.json") + if data is not None: if query_str == "all": return data else: return [snp for snp in data if snp.lower().startswith(query_str.lower())] else: - print(snps_file + " not found") + print(f"snp_list.json not found for {dataset}") return "Error: SNP list file not found" def get_gene_celltypes(dataset, gene): if dataset == "all": return "Error: Dataset is not specified." - else: - genes_file = os.path.join( - "backend", "datasets", dataset, "gene_jsons", get_gene_file_name(safe_filename(gene)) + ".json" - ) - if os.path.exists(genes_file): - with open(genes_file, "r") as f: - data = json.load(f) + gene_file = get_gene_file_name(safe_filename(gene)) + ".json" + data = ds_read_json(dataset, "gene_jsons", gene_file) - if gene.startswith("chr"): - gene_x = data.get(gene, None) - else: - gene_x = data + if data is not None: + if gene.startswith("chr"): + gene_x = data.get(gene, None) + else: + gene_x = data if gene_x: return gene_x.get("celltypes", []) else: return f"Error: Gene not found in dataset." else: - print(genes_file + " not found") + print(f"gene_jsons/{gene_file} not found for {dataset}") return "Error: Gene list file not found for the specified dataset." def get_snp_celltypes(dataset, snp): if dataset == "all": return "Error: Dataset is not specified." - else: - snps_file = os.path.join( - "backend", "datasets", dataset, "snp_jsons_merged", get_snp_group(snp) + ".json" - ) - if os.path.exists(snps_file): - with open(snps_file, "r") as f: - data = json.load(f) - snp = data.get(snp, None) + snps_file = ("snp_jsons_merged", get_snp_group(snp) + ".json") + data = ds_read_json(dataset, *snps_file) + + if data is not None: + snp = data.get(snp, None) if data and snp: return snp["celltypes"] else: return f"Error: SNP not found in dataset." else: - print(snps_file + " not found") + print(f"{'/'.join(snps_file)} not found for {dataset}") return "Error: SNP list file not found for the specified dataset." @@ -411,22 +377,16 @@ def get_snp_data_for_gene(dataset, gene, celltype=""): gene_start = gene_loc["start"] gene_end = gene_loc["end"] - celltype_mapping_file = os.path.join( - "backend", "datasets", dataset, "celltypes", "celltype_parquet.json" - ) - if not os.path.exists(celltype_mapping_file): - print(celltype_mapping_file + " not found") + celltype_mapping = ds_read_json(dataset, "celltypes", "celltype_parquet.json") + if celltype_mapping is None: + print(f"celltypes/celltype_parquet.json not found for {dataset}") return "Error: Celltype mapping file not found for the specified dataset." - with open(celltype_mapping_file, "r") as f: - celltype_mapping = json.load(f) - celltype_file = celltype_mapping.get(celltype, celltype) - data_file = os.path.join("backend", "datasets", dataset, "celltypes", celltype_file) - - if not os.path.exists(data_file): - print(data_file + " not found") + if not ds_exists(dataset, "celltypes", celltype_file): + print(f"celltypes/{celltype_file} not found for {dataset}") return "Error: QTL data file not found for the specified dataset and cell type." + data_file = ds_locator(dataset, "celltypes", celltype_file) gene_df = ( pl.scan_parquet(data_file) @@ -475,22 +435,16 @@ def get_gene_data_for_snp(dataset, is_caqtl, snp, celltype=""): return f"Error: {snp_loc}" chrom = snp_loc["chromosome"] - celltype_mapping_file = os.path.join( - "backend", "datasets", dataset, "celltypes", "celltype_parquet.json" - ) - if not os.path.exists(celltype_mapping_file): - print(celltype_mapping_file + " not found") + celltype_mapping = ds_read_json(dataset, "celltypes", "celltype_parquet.json") + if celltype_mapping is None: + print(f"celltypes/celltype_parquet.json not found for {dataset}") return "Error: Celltype mapping file not found for the specified dataset." - with open(celltype_mapping_file, "r") as f: - celltype_mapping = json.load(f) - celltype_file = celltype_mapping.get(celltype, celltype) - data_file = os.path.join("backend", "datasets", dataset, "celltypes", celltype_file) - - if not os.path.exists(data_file): - print(data_file + " not found") + if not ds_exists(dataset, "celltypes", celltype_file): + print(f"celltypes/{celltype_file} not found for {dataset}") return "Error: QTL data file not found for the specified dataset and cell type." + data_file = ds_locator(dataset, "celltypes", celltype_file) snp_df = ( pl.scan_parquet(data_file) @@ -528,14 +482,11 @@ def parse_region(region: str): ) return snp_df.to_dict(as_series=False) - gene_loc_file = os.path.join( - "backend", "datasets", dataset, "gene_locations", f"{chrom}.parquet" - ) - if not os.path.exists(gene_loc_file): - print(gene_loc_file + " not found") + if not ds_exists(dataset, "gene_locations", f"{chrom}.parquet"): + print(f"gene_locations/{chrom}.parquet not found for {dataset}") return "Error: Gene locations file not found for the specified dataset." - gene_loc_df = pl.read_parquet(gene_loc_file) + gene_loc_df = pl.read_parquet(ds_locator(dataset, "gene_locations", f"{chrom}.parquet")) if "gene_name" not in gene_loc_df.columns: return "Error: gene_locations parquet has no gene_name column for eQTL dataset." @@ -623,15 +574,12 @@ def get_meta_list(dataset, query_str="all"): def get_config_info(dataset): if dataset == "all": return "Error: Dataset is not specified." - else: - config_file = os.path.join("backend", "datasets", dataset, "dataset_info.toml") - if os.path.exists(config_file): - with open(config_file, "r") as f: - config = toml.load(f) + config = ds_read_toml(dataset, "dataset_info.toml") + if config is not None: return config else: - print(config_file + " not found") + print(f"dataset_info.toml not found for {dataset}") return "Error: Config info file not found" @@ -1040,6 +988,11 @@ def get_bw_data_exists(dataset): if dataset == "all": return "Error: Dataset is not specified." + # For remote datasets there is no listable folder; we use the bigwig + # celltype mapping file to determine if signal tracks are available + if is_remote(dataset): + return ds_exists(dataset, "bigwig", "celltype_bigwig.json") + bw_folder = os.path.join("backend", "datasets", dataset,"bigwig") if not os.path.exists(bw_folder): return False @@ -1047,16 +1000,11 @@ def get_bw_data_exists(dataset): @lru_cache(maxsize=128) def get_cached_bigwig_handle(dataset, celltype, strand=None): - celltype_mapping_file = os.path.join( - "backend", "datasets", dataset,"bigwig","celltype_bigwig.json" - ) - if not os.path.exists(celltype_mapping_file): - print(celltype_mapping_file + " not found") + celltype_mapping = ds_read_json(dataset, "bigwig", "celltype_bigwig.json") + if celltype_mapping is None: + print(f"bigwig/celltype_bigwig.json not found for {dataset}") return None - with open(celltype_mapping_file, "r") as f: - celltype_mapping = json.load(f) - entry = celltype_mapping.get(celltype, celltype) if isinstance(entry, str): @@ -1073,13 +1021,7 @@ def get_cached_bigwig_handle(dataset, celltype, strand=None): print(f"No matching bigWig for celltype={celltype}, strand={strand}") return None - bw_path = os.path.join( - "backend", - "datasets", - dataset, - "bigwig", - celltype_file, - ) + bw_path = ds_locator(dataset, "bigwig", celltype_file) try: return pyBigWig.open(bw_path) @@ -1110,16 +1052,11 @@ def get_region_signal_data(dataset, chromosome, start, end, celltype="", bin_siz try: # Read mapping once here to know if we have plus/minus - celltype_mapping_file = os.path.join( - "backend", "datasets", dataset, "bigwig", "celltype_bigwig.json" - ) - if not os.path.exists(celltype_mapping_file): - print(celltype_mapping_file + " not found") + celltype_mapping = ds_read_json(dataset, "bigwig", "celltype_bigwig.json") + if celltype_mapping is None: + print(f"bigwig/celltype_bigwig.json not found for {dataset}") return "Error: BigWig mapping file not found" - with open(celltype_mapping_file, "r") as f: - celltype_mapping = json.load(f) - entry = celltype_mapping.get(celltype, celltype) if strand in {"+", "-"}: @@ -1270,19 +1207,12 @@ def get_gene_annotation_info(dataset): if dataset == "all": return {} - ann_file = os.path.join( - "backend", "datasets", dataset, "gene_locations", "gene_annotations.toml" - ) - if not os.path.exists(ann_file): + try: + cfg = ds_read_toml(dataset, "gene_locations", "gene_annotations.toml") + except Exception as e: + print(f"Error reading gene_annotations.toml for {dataset}: {e}") return {} - with open(ann_file, "r") as f: - try: - cfg = toml.load(f) - except Exception as e: - print(f"Error reading {ann_file}: {e}") - return {} - return cfg or {} @@ -1297,16 +1227,11 @@ def get_celltype_list(dataset): if dataset == "all": return "Error: Dataset is not specified." - celltype_mapping_file = os.path.join( - "backend", "datasets", dataset,'celltypes', "celltype_parquet.json" - ) - if not os.path.exists(celltype_mapping_file): - print(celltype_mapping_file + " not found") + celltype_mapping = ds_read_json(dataset, "celltypes", "celltype_parquet.json") + if celltype_mapping is None: + print(f"celltypes/celltype_parquet.json not found for {dataset}") return f"Error: Celltype mapping file not found for the dataset" - with open(celltype_mapping_file, "r") as f: - celltype_mapping = json.load(f) - return list(celltype_mapping.keys()) @@ -1314,19 +1239,59 @@ def get_bigwig_celltype_list(dataset): if dataset == "all": return "Error: Dataset is not specified." - celltype_mapping_file = os.path.join( - "backend", "datasets", dataset, "bigwig", "celltype_bigwig.json" - ) - if not os.path.exists(celltype_mapping_file): - print(celltype_mapping_file + " not found") + celltype_mapping = ds_read_json(dataset, "bigwig", "celltype_bigwig.json") + if celltype_mapping is None: + print(f"bigwig/celltype_bigwig.json not found for {dataset}") return "Error: BigWig celltype mapping file not found for the dataset" - with open(celltype_mapping_file, "r") as f: - celltype_mapping = json.load(f) - return list(celltype_mapping.keys()) +def inspect_dataset(dataset): + if not dataset or dataset == "all": + return {"reachable": False, "error": "No dataset specified."} + + try: + has_qtl = bool(ds_exists(dataset, "celltypes", "celltype_parquet.json")) + has_bw = get_bw_data_exists(dataset) + has_bw = bool(has_bw) if not isinstance(has_bw, str) else False + + config = get_config_info(dataset) + datatype = "" + info = {} + if isinstance(config, dict): + datatype = (config.get("datasetfile", {}) or {}).get("datatype", "") or "" + info = config.get("dataset", {}) or {} + + is_caqtl = datatype.lower().startswith("ca") + + # Fall back to structural heuristics when there is no dataset_info.toml. + if not datatype: + if has_qtl: + # caQTL groups gene jsons by chromosome (chr1.json ...), eQTL uses + # one file per gene symbol. + is_caqtl = bool(ds_exists(dataset, "gene_jsons", "chr1.json")) + datatype = "caQTL" if is_caqtl else "eQTL" + elif has_bw: + datatype = "scATACseq" + + reachable = bool(has_qtl or has_bw or isinstance(config, dict)) + + return { + "reachable": reachable, + "dataset_id": dataset, + "assay": datatype, + "is_caqtl": is_caqtl, + "has_qtl": has_qtl, + "has_bw": has_bw, + "dataset_name": info.get("dataset_name", "") if info else "", + "info": info, + } + except Exception as e: + # Includes RemoteDatasetError (unsafe/blocked URL) and transport errors + return {"reachable": False, "dataset_id": dataset, "error": str(e)} + + def get_exon_structure_in_chromosome(dataset, chromosome, start, end, max_exons=2000): if dataset == "all": return "Error: Dataset is not specified." @@ -1341,9 +1306,9 @@ def get_exon_structure_in_chromosome(dataset, chromosome, start, end, max_exons= # return "Error: Annotation BED file not specified for the dataset." bed_rel = "transcript_annotation.bed" - bed_path = os.path.join("backend", "datasets", dataset, bed_rel) - if not os.path.exists(bed_path): - print(bed_path + " not found") + bed_stream = ds_text_stream(dataset, bed_rel) + if bed_stream is None: + print(f"{bed_rel} not found for {dataset}") return "Error: Annotation BED file not found for the specified dataset." chrom_list = [] @@ -1363,7 +1328,7 @@ def get_exon_structure_in_chromosome(dataset, chromosome, start, end, max_exons= truncated = False import csv - with open(bed_path) as f: + with bed_stream as f: reader = csv.reader(f, delimiter="\t") for row in reader: if len(row) < 12: diff --git a/backend/funcs/remote_capabilities.py b/backend/funcs/remote_capabilities.py new file mode 100644 index 0000000..ca46465 --- /dev/null +++ b/backend/funcs/remote_capabilities.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +import importlib +import logging +from dataclasses import dataclass, field +from typing import Callable, List, Optional + +logger = logging.getLogger("vizit.remote_capabilities") + + +@dataclass +class CapabilityResult: + name: str # short id + required_for: str # description of the feature + ok: Optional[bool] # True = available, False = missing, None = could not determine + detail: str # description of the check and its result + fix_hint: str = "" # how to install + + +# --------------------------------------------------------------------------- # +# Registry +# --------------------------------------------------------------------------- # +CapabilityCheck = Callable[[], CapabilityResult] +_CHECKS: List[CapabilityCheck] = [] + + +def register_capability_check(fn: CapabilityCheck) -> CapabilityCheck: + _CHECKS.append(fn) + return fn + + +def run_capability_checks() -> List[CapabilityResult]: + results: List[CapabilityResult] = [] + for fn in _CHECKS: + try: + results.append(fn()) + except Exception as exc: + results.append( + CapabilityResult( + name=getattr(fn, "__name__", "unknown"), + required_for="unknown", + ok=None, + detail=f"capability check raised {type(exc).__name__}: {exc}", + ) + ) + return results + + +# --------------------------------------------------------------------------- # +# Probes +# --------------------------------------------------------------------------- # +def _module_available(modname: str) -> bool: + try: + importlib.import_module(modname) + return True + except Exception: + return False + + +@register_capability_check +def check_polars_http() -> CapabilityResult: + have_fsspec = _module_available("fsspec") + have_aiohttp = _module_available("aiohttp") + ok = have_fsspec and have_aiohttp + if ok: + detail = "fsspec + aiohttp present; polars can read parquet/CSV over http(s)." + fix = "" + else: + missing = [ + m + for m, present in (("fsspec", have_fsspec), ("aiohttp", have_aiohttp)) + if not present + ] + detail = ( + f"missing {', '.join(missing)}; polars cannot read parquet/CSV from " + "remote URLs (gene/SNP locations, QTL parquet, GWAS tables)." + ) + fix = "pip install fsspec aiohttp" + return CapabilityResult( + name="polars-http", + required_for="remote QTL parquet / gene & SNP locations / GWAS tables", + ok=ok, + detail=detail, + fix_hint=fix, + ) + + +@register_capability_check +def check_pybigwig_remote() -> CapabilityResult: + try: + import pyBigWig + except Exception as exc: + return CapabilityResult( + name="pybigwig-remote", + required_for="remote BigWig signal tracks", + ok=None, + detail=f"pyBigWig import failed: {type(exc).__name__}: {exc}", + fix_hint="pip install pyBigWig", + ) + + ok = bool(getattr(pyBigWig, "remote", 0)) + if ok: + detail = ( + "pyBigWig was built with libcurl; remote .bw/.bigWig URLs are supported." + ) + fix = "" + else: + detail = ( + "pyBigWig was built WITHOUT libcurl; " + "remote BigWig signal tracks will fail to open." + ) + fix = ( + "Install a curl-enabled build, e.g. install libcurl " + "and then run `pip install --no-binary pyBigWig pyBigWig`" + ) + return CapabilityResult( + name="pybigwig-remote", + required_for="remote BigWig signal tracks", + ok=ok, + detail=detail, + fix_hint=fix, + ) + + +# --------------------------------------------------------------------------- # +# Startup +# --------------------------------------------------------------------------- # +def warn_on_missing_remote_capabilities() -> List[CapabilityResult]: + results = run_capability_checks() + missing = [r for r in results if r.ok is False] + unknown = [r for r in results if r.ok is None] + + if not missing and not unknown: + logger.info( + "Remote datasets enabled; all %d optional capabilities available.", + len(results), + ) + return results + + logger.warning( + "Remote datasets are ENABLED but %d capability(ies) are unavailable. " + "Affected features will error for remote datasets:", + len(missing) + len(unknown), + ) + for r in missing + unknown: + state = "MISSING" if r.ok is False else "UNKNOWN" + logger.warning(" [%s] %s — required for: %s", state, r.name, r.required_for) + logger.warning(" %s", r.detail) + if r.fix_hint: + logger.warning(" fix: %s", r.fix_hint) + return results diff --git a/backend/funcs/remote_io.py b/backend/funcs/remote_io.py new file mode 100644 index 0000000..67c4dca --- /dev/null +++ b/backend/funcs/remote_io.py @@ -0,0 +1,241 @@ +import io +import ipaddress +import json +import os +import socket +from functools import lru_cache +from urllib.parse import urljoin, urlsplit, quote + +import requests +import toml + +from backend.settings import settings + +DATASETS_DIR = os.path.join("backend", "datasets") + +# Networking limits for remote fetches. +REMOTE_TIMEOUT = 30 # seconds per request +MAX_REMOTE_BYTES = 1024 * 1024 * 1024 # 1 GiB cap for whole-file downloads +MAX_REDIRECTS = 3 + + +class RemoteDatasetError(Exception): + pass + + +def is_remote(dataset) -> bool: + return isinstance(dataset, str) and ( + dataset.startswith("http://") or dataset.startswith("https://") + ) + + +# --------------------------------------------------------------------------- # +# SSRF protection +# --------------------------------------------------------------------------- # +def _allowed_hosts() -> set: + raw = getattr(settings, "remote_dataset_allowed_hosts", "") or "" + return {h.strip().lower() for h in raw.split(",") if h.strip()} + + +def _ip_is_public(ip_text: str) -> bool: + try: + ip = ipaddress.ip_address(ip_text) + except ValueError: + return False + + # Unwrap IPv4-mapped / 6to4-style addresses so 127.0.0.1 can't be hidden as ::ffff:127.0.0.1 + if ip.version == 6 and getattr(ip, "ipv4_mapped", None) is not None: + ip = ip.ipv4_mapped + if ( + ip.is_private + or ip.is_loopback + or ip.is_link_local + or ip.is_multicast + or ip.is_reserved + or ip.is_unspecified + ): + return False + return bool(ip.is_global) + + +def _resolve_ips(host: str) -> set: + infos = socket.getaddrinfo(host, None) + return {info[4][0] for info in infos} + + +def assert_safe_url(url: str) -> str: + if not getattr(settings, "allow_remote_datasets", True): + raise RemoteDatasetError("Remote datasets are disabled on this server.") + + parts = urlsplit(url) + if parts.scheme not in ("http", "https"): + raise RemoteDatasetError(f"Unsupported URL scheme: {parts.scheme!r}") + host = parts.hostname + if not host: + raise RemoteDatasetError("Remote dataset URL has no host.") + + allow = _allowed_hosts() + if allow and host.lower() not in allow: + raise RemoteDatasetError(f"Host '{host}' is not in the allowed-hosts list.") + + if getattr(settings, "remote_dataset_allow_private", False): + return url + + try: + ips = _resolve_ips(host) + except socket.gaierror as exc: + raise RemoteDatasetError(f"Could not resolve host '{host}': {exc}") + if not ips: + raise RemoteDatasetError(f"Could not resolve host '{host}'.") + for ip in ips: + if not _ip_is_public(ip): + raise RemoteDatasetError( + f"Refusing to access non-public address {ip} (host '{host}')." + ) + return url + + +# --------------------------------------------------------------------------- # +# HTTP helpers +# --------------------------------------------------------------------------- # +def _request(method: str, url: str, **kwargs): + """Issue a request, re-validating the target on every redirect hop.""" + assert_safe_url(url) + current = url + for _ in range(MAX_REDIRECTS + 1): + resp = requests.request( + method, + current, + allow_redirects=False, + timeout=REMOTE_TIMEOUT, + **kwargs, + ) + if resp.status_code in (301, 302, 303, 307, 308): + location = resp.headers.get("Location") + resp.close() + if not location: + raise RemoteDatasetError("Redirect response without a Location header.") + current = urljoin(current, location) + assert_safe_url(current) + continue + return resp + raise RemoteDatasetError("Too many redirects while fetching remote dataset file.") + + +def _remote_exists(url: str) -> bool: + try: + resp = _request("GET", url, stream=True, headers={"Range": "bytes=0-0"}) + except RemoteDatasetError: + raise + except requests.RequestException: + return False + try: + return resp.status_code < 400 + finally: + resp.close() + + +def clear_remote_cache() -> None: + _fetch_bytes_cached.cache_clear() + + +@lru_cache(maxsize=512) +def _fetch_bytes_cached(url: str) -> bytes: + try: + resp = _request("GET", url, stream=True) + except requests.RequestException as exc: + raise RemoteDatasetError(f"Failed to fetch {url}: {exc}") + + with resp: + if resp.status_code == 404: + raise FileNotFoundError(url) + if resp.status_code >= 400: + raise RemoteDatasetError( + f"Remote returned HTTP {resp.status_code} for {url}" + ) + + total = 0 + chunks = [] + for chunk in resp.iter_content(chunk_size=1 << 16): + if not chunk: + continue + total += len(chunk) + if total > MAX_REMOTE_BYTES: + raise RemoteDatasetError( + f"Remote file {url} exceeds the {MAX_REMOTE_BYTES} byte limit." + ) + chunks.append(chunk) + return b"".join(chunks) + + +# --------------------------------------------------------------------------- # +# Path / URL building +# --------------------------------------------------------------------------- # +def _local_path(dataset: str, parts) -> str: + return os.path.join(DATASETS_DIR, dataset, *parts) + + +def _remote_url(dataset: str, parts) -> str: + base = dataset.rstrip("/") + suffix = "/".join(quote(str(p), safe="") for p in parts) + return f"{base}/{suffix}" if suffix else base + + +def ds_locator(dataset: str, *parts) -> str: + if is_remote(dataset): + return assert_safe_url(_remote_url(dataset, parts)) + return _local_path(dataset, parts) + + +def ds_exists(dataset: str, *parts) -> bool: + if is_remote(dataset): + return _remote_exists(_remote_url(dataset, parts)) + return os.path.exists(_local_path(dataset, parts)) + + +def ds_read_bytes(dataset: str, *parts): + if is_remote(dataset): + try: + return _fetch_bytes_cached(_remote_url(dataset, parts)) + except FileNotFoundError: + return None + path = _local_path(dataset, parts) + if not os.path.exists(path): + return None + with open(path, "rb") as fh: + return fh.read() + + +def ds_read_text(dataset: str, *parts, encoding="utf-8"): + data = ds_read_bytes(dataset, *parts) + if data is None: + return None + return data.decode(encoding) + + +def ds_read_json(dataset: str, *parts): + text = ds_read_text(dataset, *parts) + if text is None: + return None + return json.loads(text) + + +def ds_read_toml(dataset: str, *parts): + text = ds_read_text(dataset, *parts) + if text is None: + return None + return toml.loads(text) + + +def ds_text_stream(dataset: str, *parts): + text = ds_read_text(dataset, *parts) + if text is None: + return None + return io.StringIO(text) + + +def ds_bytes_stream(dataset: str, *parts): + data = ds_read_bytes(dataset, *parts) + if data is None: + return None + return io.BytesIO(data) diff --git a/backend/main.py b/backend/main.py index 3857b2a..97a367b 100644 --- a/backend/main.py +++ b/backend/main.py @@ -1,13 +1,42 @@ from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse +from starlette.requests import Request from backend.db import create_db_and_tables -from backend.routes import db_routes, api_routes, visium_routes, qtl_routes, dm_routes,signal_routes +from backend.routes import ( + db_routes, + api_routes, + visium_routes, + qtl_routes, + dm_routes, + signal_routes, +) +from backend.funcs.remote_io import RemoteDatasetError +from backend.funcs.remote_capabilities import ( + warn_on_missing_remote_capabilities, + run_capability_checks, +) from backend.settings import settings app = FastAPI(debug=settings.debug) + +def check_remote_capabilities_on_startup(): + # Only relevant remote datasets is enabled + if settings.allow_remote_datasets: + warn_on_missing_remote_capabilities() + + +@app.exception_handler(RemoteDatasetError) +async def remote_dataset_error_handler(request: Request, exc: RemoteDatasetError): + # A remote (client-supplied) dataset URL was disallowed or unreachable. + return JSONResponse( + status_code=400, content={"detail": f"Remote dataset error: {exc}"} + ) + + app.add_middleware( CORSMiddleware, allow_origins=[ @@ -19,11 +48,31 @@ ) app.add_event_handler("startup", create_db_and_tables) +app.add_event_handler("startup", check_remote_capabilities_on_startup) + @app.get("/") async def root(): return "Hello, Welcome to VizIt!" + +@app.get("/serverconfig") +async def server_config(): + caps = run_capability_checks() if settings.allow_remote_datasets else [] + return { + "allow_remote_datasets": settings.allow_remote_datasets, + "remote_capabilities": [ + { + "name": c.name, + "required_for": c.required_for, + "ok": c.ok, + "detail": c.detail, + } + for c in caps + ], + } + + app.include_router(db_routes.router, prefix="/db") app.include_router(api_routes.router, prefix="/api") app.include_router(visium_routes.router, prefix="/visium") @@ -33,6 +82,7 @@ async def root(): if __name__ == "__main__": import uvicorn + print("Starting FastAPI server on port", settings.uvicorn_port) uvicorn.run(app, host=settings.uvicorn_host, port=settings.uvicorn_port) diff --git a/backend/requirements.txt b/backend/requirements.txt index c9e7a88..c0d0966 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -11,5 +11,6 @@ polars == 1.31.0 fastparquet == 2024.11.0 types-pytz == 2025.2.* pydantic-settings == 2.10.1 -pyBigWig == 0.3.24 +pyBigWig == 0.3.24 # pyBigWig must be built against libcurl so it can stream remote (http/https) +requests == 2.32.* diff --git a/backend/routes/dm_routes.py b/backend/routes/dm_routes.py index f246e50..594c605 100644 --- a/backend/routes/dm_routes.py +++ b/backend/routes/dm_routes.py @@ -310,48 +310,48 @@ def check_toml_file(toml_data): return {"message": "Config file is valid", "success": True} -@router.get("/refreshdatabase") -async def refreshdatabase(session: Session = Depends(get_session)): - try: - ## loop through all datasets - for dataset_i in os.listdir("backend/datasets"): - dataset_path = f"backend/datasets/{dataset_i}" - ## load dataset info - dataset_info_file = f"{dataset_path}/dataset_info.toml" - if not os.path.exists(dataset_info_file): - continue - - with open(f"{dataset_path}/dataset_info.toml", 'r') as f: - dataset_info = toml.load(f) - - ## check if dataset configuration is valid - check_result = check_toml_file(dataset_info) - if not check_result["success"]: - return check_result - - print("=======insert info into database==========") - study_dict= dataset_info["study"] - study_dict["study_id"] = study_dict["study_name"] - study = Study(**study_dict) - - dataset_dict = dataset_info["dataset"] - dataset_dict["dataset_id"] = dataset_dict["dataset_name"] - dataset_dict["assay"] = dataset_info["datasetfile"]["datatype"] - dataset_dict["study_id"] = study_dict["study_id"] - dataset_dict["dataset_file"] = dataset_info["datasetfile"]["file"] - dataset = Dataset(**dataset_dict) - - insert_study(study, session) - insert_dataset(dataset, session) - - if not(dataset_dict["sample_sheet"] == "None" or dataset_dict["sample_sheet"] == ""): - sample_sheet_path = f"backend/SampleSheets/{dataset_dict['sample_sheet']}" - shutil.copyfile(sample_sheet_path, f"{dataset_path}/{dataset_dict['sample_sheet']}") - import_sample_sheet(sample_sheet_path, session) - - return {"message": "Database refreshed successfully", "success": True} - except Exception as e: - return {"message": "Error: " + str(e), "success": False} +# @router.get("/refreshdatabase") +# async def refreshdatabase(session: Session = Depends(get_session)): +# try: +# ## loop through all datasets +# for dataset_i in os.listdir("backend/datasets"): +# dataset_path = f"backend/datasets/{dataset_i}" +# ## load dataset info +# dataset_info_file = f"{dataset_path}/dataset_info.toml" +# if not os.path.exists(dataset_info_file): +# continue + +# with open(f"{dataset_path}/dataset_info.toml", 'r') as f: +# dataset_info = toml.load(f) + +# ## check if dataset configuration is valid +# check_result = check_toml_file(dataset_info) +# if not check_result["success"]: +# return check_result + +# print("=======insert info into database==========") +# study_dict= dataset_info["study"] +# study_dict["study_id"] = study_dict["study_name"] +# study = Study(**study_dict) + +# dataset_dict = dataset_info["dataset"] +# dataset_dict["dataset_id"] = dataset_dict["dataset_name"] +# dataset_dict["assay"] = dataset_info["datasetfile"]["datatype"] +# dataset_dict["study_id"] = study_dict["study_id"] +# dataset_dict["dataset_file"] = dataset_info["datasetfile"]["file"] +# dataset = Dataset(**dataset_dict) + +# insert_study(study, session) +# insert_dataset(dataset, session) + +# if not(dataset_dict["sample_sheet"] == "None" or dataset_dict["sample_sheet"] == ""): +# sample_sheet_path = f"backend/SampleSheets/{dataset_dict['sample_sheet']}" +# shutil.copyfile(sample_sheet_path, f"{dataset_path}/{dataset_dict['sample_sheet']}") +# import_sample_sheet(sample_sheet_path, session) + +# return {"message": "Database refreshed successfully", "success": True} +# except Exception as e: +# return {"message": "Error: " + str(e), "success": False} @router.delete("/deletedataset") diff --git a/backend/routes/qtl_routes.py b/backend/routes/qtl_routes.py index 3bcc1f5..36a5e27 100644 --- a/backend/routes/qtl_routes.py +++ b/backend/routes/qtl_routes.py @@ -3,6 +3,7 @@ # from backend.funcs.get_data import * +from backend.funcs.remote_io import is_remote, clear_remote_cache from backend.funcs.get_data import ( get_qtl_gene_list, get_qtl_snp_list, @@ -19,6 +20,7 @@ get_gwas_datasets, get_gwas_in_chromosome, get_gencode_version, + inspect_dataset, ) router = APIRouter() @@ -29,6 +31,14 @@ async def read_root(): return {"Message": "Hello QTL."} +@router.get("/inspectdataset") +async def inspectdataset(request: Request): + dataset_id = request.query_params.get("dataset") + if is_remote(dataset_id): + clear_remote_cache() + return inspect_dataset(dataset_id) + + @router.get("/getgenelocation") async def getgenelocation(request: Request): print("getgenelocation() called================") diff --git a/backend/settings.py b/backend/settings.py index 74abff7..235a0dd 100644 --- a/backend/settings.py +++ b/backend/settings.py @@ -5,6 +5,10 @@ class Settings(BaseSettings): uvicorn_host: str = "0.0.0.0" debug: bool = False + allow_remote_datasets: bool = True + remote_dataset_allowed_hosts: str = "" + remote_dataset_allow_private: bool = False + class Config: env_file = ".env" # 可选：已手动 load_dotenv 也可以省略 diff --git a/docs/prepare_dataset/scrnaseq.md b/docs/prepare_dataset/scrnaseq.md index 1dc938c..f6e2f4d 100644 --- a/docs/prepare_dataset/scrnaseq.md +++ b/docs/prepare_dataset/scrnaseq.md @@ -296,7 +296,7 @@ After running the pipeline, there will be a dataset folder that contains all nec * Upload the dataset folder named '<dataset_name>' to the server at __'backend/datasets'__. * Put the dataset configuration file named __'dataset_info.toml'__ to your dataset folder at __'backend/datasets/<dataset_name>'__. * Upload __samplesheet file__ to the server at __'backend/SampleSheet'__. -* Refresh the database: Go to __'Datasets Management Page '(DATASETS -> +ADD DATASET)__ and click __'REFRESH DB'__. +* Refresh the database: on the server, from the repository root, run `python -m backend.db_utils.refresh_db`. diff --git a/docs/prepare_dataset/visiumst.md b/docs/prepare_dataset/visiumst.md index 5bf78a2..e11349b 100644 --- a/docs/prepare_dataset/visiumst.md +++ b/docs/prepare_dataset/visiumst.md @@ -304,7 +304,7 @@ After running the pipeline, there will be a dataset folder that contains all nec * Upload the dataset folder named '<dataset_name>' to the server at __'backend/datasets'__. * Put the dataset configuration file named __'dataset_info.toml'__ to your dataset folder at __'backend/datasets/<dataset_name>'__. * Upload __samplesheet file__ to the server at __'backend/SampleSheet'__. -* Refresh the database: Go to __'Datasets Management Page '(DATASETS -> +ADD DATASET)__ and click __'REFRESH DB'__. +* Refresh the database: on the server, from the repository root, run `python -m backend.db_utils.refresh_db`. diff --git a/docs/prepare_dataset/xqtl.md b/docs/prepare_dataset/xqtl.md index 1a7f20c..089eba2 100644 --- a/docs/prepare_dataset/xqtl.md +++ b/docs/prepare_dataset/xqtl.md @@ -229,7 +229,16 @@ After running the pipeline, there will be a dataset folder that contains all nec * The files and folders within the `your_dataset_name` folder are necessary for upload. * Upload the dataset folder named '<dataset_name>' to the server at __'backend/datasets'__. * Put the dataset configuration file named __'dataset_info.toml'__ to your dataset folder at __'backend/datasets/<dataset_name>'__. -* Refresh the database: Go to __'Datasets Management Page '(DATASETS -> +ADD DATASET)__ and click __'REFRESH DB'__. +* Refresh the database: on the server, from the repository root, run `python -m backend.db_utils.refresh_db`. + +Alternatively, you can host the dataset folder on a web server and add it from the +__'DATASETS -> + ADD DATASET'__ dialog with the __'Load from URL'__ option. The dataset is then registered +in your browser only, and no database refresh is needed. This requires: + +* A web server that supports __HTTP range requests__ (nginx, Apache, Caddy, S3, ...). Python's `http.server` does __not__ work since it ignores `Range` headers. +* A URL that is reachable __from the machine running the VizIt backend__. +* `REMOTE_DATASET_ALLOW_PRIVATE=true` in `backend/.env` if the dataset is served from a private or loopback + address, which is otherwise refused. diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 1ed2cd9..4151cc0 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -22,7 +22,7 @@ import GenomicRegionView from "./pages/GenomicRegionView"; import XDatasetsView from "./pages/XDatasets"; import CellTypesView from "./pages/CellTypesView"; import ViewsHome from "./pages/ViewsHome"; -import DatasetManagePage from "./pages/DatasetManage"; +// import DatasetManagePage from "./pages/DatasetManage"; import LayerView from "./pages/LayerView/index.jsx"; import HowToUse from "./pages/Help/HowToUse.jsx"; import FAQPage from "./pages/Help/FAQ.jsx"; @@ -113,7 +113,7 @@ function App() { }/> - }/> + {/*}/>*/} }/> diff --git a/frontend/src/api/api.js b/frontend/src/api/api.js index 9d274c6..f1066ad 100644 --- a/frontend/src/api/api.js +++ b/frontend/src/api/api.js @@ -3,6 +3,16 @@ import axios from "axios"; const BASE_URL = import.meta.env.VITE_BACKEND_URL; const API_URL = `${BASE_URL}/api`; +export const getServerConfig = async () => { + try { + const response = await axios.get(`${BASE_URL}/serverconfig`); + return response.data; + } catch (error) { + console.error("Error getServerConfig:", error); + throw error; + } +} + export const getHomeData = async () => { try { const response = await axios.get(`${API_URL}/gethomedata`); diff --git a/frontend/src/api/qtl.js b/frontend/src/api/qtl.js index c57a0b9..e083518 100644 --- a/frontend/src/api/qtl.js +++ b/frontend/src/api/qtl.js @@ -3,6 +3,18 @@ import axios from "axios"; const BASE_URL = import.meta.env.VITE_BACKEND_URL; const QTL_URL = `${BASE_URL}/qtl`; +export const inspectDataset = async (dataset) => { + try { + const response = await axios.get(`${QTL_URL}/inspectdataset`, { + params: { dataset: dataset }, + }); + return response.data; + } catch (error) { + console.error("Error inspectDataset:", error); + throw error; + } +}; + export const getGeneLocation = async (dataset, gene) => { try { const response = await axios.get(`${QTL_URL}/getgenelocation`, { diff --git a/frontend/src/pages/DatasetDemos/SingleCellDemo.jsx b/frontend/src/pages/DatasetDemos/SingleCellDemo.jsx index ca62efb..aafb5f7 100644 --- a/frontend/src/pages/DatasetDemos/SingleCellDemo.jsx +++ b/frontend/src/pages/DatasetDemos/SingleCellDemo.jsx @@ -390,7 +390,7 @@ genes = [ "SNCA",...] ## List of default gene names
  • 3. Upload the dataset folder named '<dataset_name>' to the server at 'backend/datasets'.
  • 4. Put the dataset configuration file named 'dataset_info.toml' to your dataset folder at 'backend/datasets/<dataset_name>'.
  • 5. Upload samplesheet file to the server at 'backend/SampleSheet'.
    • -
  • 6. Refresh the database: Go to 'Datasets Management Page '(DATASETS -> +ADD DATASET) and click 'REFRESH DB'.
    • +
  • 6. Refresh the database on the server, from the repository root: python -m backend.db_utils.refresh_db

    • diff --git a/frontend/src/pages/DatasetDemos/VisiumSTDemo.jsx b/frontend/src/pages/DatasetDemos/VisiumSTDemo.jsx index ab6f059..dbe2d23 100644 --- a/frontend/src/pages/DatasetDemos/VisiumSTDemo.jsx +++ b/frontend/src/pages/DatasetDemos/VisiumSTDemo.jsx @@ -387,7 +387,7 @@ genes = [ "SNCA",...] ## List of default gene names
  • 3. Upload the dataset folder named '<dataset_name>' to the server at 'backend/datasets'.
  • 4. Put the dataset configuration file named 'dataset_info.toml' to your dataset folder at 'backend/datasets/<dataset_name>'.
  • 5. Upload samplesheet file to the server at 'backend/SampleSheet'.
    • -
  • 6. Refresh the database: Go to 'Datasets Management Page '(DATASETS -> +ADD DATASET) and click 'REFRESH DB'.
    • +
  • 6. Refresh the database on the server, from the repository root: python -m backend.db_utils.refresh_db

    • diff --git a/frontend/src/pages/DatasetManage/index.jsx b/frontend/src/pages/DatasetManage/index.jsx index 738d36f..bb5e3fa 100644 --- a/frontend/src/pages/DatasetManage/index.jsx +++ b/frontend/src/pages/DatasetManage/index.jsx @@ -1,7 +1,7 @@ import {useState, useRef, useEffect} from 'react'; import {useSearchParams} from "react-router-dom"; -import {Box, Stepper, Step, StepLabel, Button, Paper, Typography, Divider,} from '@mui/material'; +import {Box, Stepper, Step, StepLabel, /* Button, */ Paper, Typography, Divider,} from '@mui/material'; import ExtractInfo from './ExtractInfo'; import ExtractInfoProcess from "./ExtractInfoProcess.jsx"; @@ -15,7 +15,8 @@ const steps = ['Setup dataset', 'Extracting data', 'Prepare metadata', 'Running const DatasetManage = () => { const {setDatasetName, extractData, isProcessing} = useDatasetManageStore(); - const {prepareMetaData,refreshDatabase} = useDatasetManageStore(); + const {prepareMetaData} = useDatasetManageStore(); + // const {refreshDatabase} = useDatasetManageStore(); // Get all the pre-selected values const [queryParams, setQueryParams] = useSearchParams(); @@ -88,14 +89,14 @@ const DatasetManage = () => { newParams.set("stepidx", activeStep - 1) setQueryParams(newParams); }; - const handleImportDataInfo = () => { - try { - const response = refreshDatabase(); - } catch (error) { - console.error('Submission failed:', error); - alert('Failed to submit dataset info.'); - } - }; + // const handleImportDataInfo = () => { + // try { + // const response = refreshDatabase(); + // } catch (error) { + // console.error('Submission failed:', error); + // alert('Failed to submit dataset info.'); + // } + // }; const getStepContent = (step) => { switch (step) { @@ -124,15 +125,15 @@ const DatasetManage = () => { {steps.map((label) => ({label}))} {/* Import Button */} - - - + {/**/} + {/* */} + {/* Refresh DB*/} + {/* */} + {/**/} @@ -142,7 +143,7 @@ const DatasetManage = () => { Option 1: Use command line to process dataset
    1. Process the dataset using the provided scripts (Customize the scripts for your dataset)
    2. Upload the processed dataset to "backend > datasets" folder, -
    3. Click the "Refresh DB" button +
    3. Refresh the database on the server: python -m backend.db_utils.refresh_db {/**/} diff --git a/frontend/src/pages/Datasets/AddDatasetDialog.jsx b/frontend/src/pages/Datasets/AddDatasetDialog.jsx new file mode 100644 index 0000000..38aaf31 --- /dev/null +++ b/frontend/src/pages/Datasets/AddDatasetDialog.jsx @@ -0,0 +1,344 @@ +import { useEffect, useRef, useState } from "react"; +import { + Dialog, + DialogTitle, + DialogContent, + DialogActions, + Button, + Box, + Typography, + Stepper, + Step, + StepLabel, + Radio, + RadioGroup, + FormControlLabel, + Paper, + Alert, + AlertTitle, + Link, + Divider, +} from "@mui/material"; +import PropTypes from "prop-types"; + +import RemoteUrlForm from "./RemoteUrlForm.jsx"; +import useServerConfigStore from "../../store/ServerConfigStore.js"; + +const REFRESH_DB_COMMAND = "python -m backend.db_utils.refresh_db"; + +const LOCAL = "local"; +const SERVER = "server"; + +const CodeBlock = ({ children }) => ( + + {children} + +); + +CodeBlock.propTypes = { children: PropTypes.node }; + +const InlineCode = ({ children }) => ( + + {children} + +); + +InlineCode.propTypes = { children: PropTypes.node }; + +const AddDatasetDialog = ({ open, onClose }) => { + const { allowRemoteDatasets } = useServerConfigStore(); + const remoteEnabled = allowRemoteDatasets !== false; + + const [activeStep, setActiveStep] = useState(0); + const [choice, setChoice] = useState(remoteEnabled ? LOCAL : SERVER); + const [busy, setBusy] = useState(false); + const remoteFormRef = useRef(); + + useEffect(() => { + if (!remoteEnabled) setChoice(SERVER); + }, [remoteEnabled]); + + const steps = + choice === SERVER + ? ["Preprocess dataset", "Add dataset", "Refresh database"] + : ["Preprocess dataset", "Add dataset"]; + + const handleClose = () => { + setActiveStep(0); + setChoice(remoteEnabled ? LOCAL : SERVER); + setBusy(false); + onClose(); + }; + + const handleNext = () => setActiveStep((prev) => prev + 1); + const handleBack = () => setActiveStep((prev) => prev - 1); + + const getStepContent = (step) => { + switch (step) { + case 0: + return ( + <> + + Process your raw data into the VizIt dataset layout + using the provided{" "} + + scripts + {" "} + (customizable per dataset). + + + When the pipeline finishes you will have a dataset + folder containing dataset_info.toml and + the data files the views read ( + celltypes/,{" "} + bigwig/,{" "} + gene_jsons/, …). Continue + once that folder is ready. + + + ); + case 1: + return ( + <> + + Choose how the dataset should be added. + + setChoice(e.target.value)} + > + + } + disabled={!remoteEnabled} + label={ + + + Load from URL + + + The dataset is hosted on an HTTP + server and registered in your + browser only. Nothing is + uploaded and no shell access to + this server is needed. + + + } + /> + {!remoteEnabled && ( + + Loading datasets from a URL is disabled + on this server. + + )} + {choice === LOCAL && remoteEnabled && ( + + + + + Requirements for the hosting + server + + +
  • + It must support HTTP range + requests (nginx, Apache, + Caddy, S3, …). Python's{" "} + + http.server + {" "} + will not work since it ignores + Range headers. +
    • +
  • + The URL must be reachable + from machine running VizIt. +
    • +
  • + Private or loopback + addresses are rejected + unless{" "} + + REMOTE_DATASET_ALLOW_PRIVATE=true + {" "} + is set in{" "} + + backend/.env + + . +
    • +     +     + +     + )} +     + + + } + label={ + + + Add on the server + + + For server administrators: copy + the dataset into the backend so + it is listed for everyone. Needs + shell access to the server. + + + } + /> + +     + + ); + case 2: + return ( + <> + + On the server, place the dataset and then refresh + the metadata database: + + + 1. Copy the dataset folder to{" "} + + backend/datasets/<dataset_name>/ + + , with its{" "} + dataset_info.toml inside. +
    + 2. Copy the sample sheet (if the dataset has one) to{" "} + backend/SampleSheets/. +
    + 3. Run this from the repository root: +     + {REFRESH_DB_COMMAND} + + 4. Reload this page. The dataset should be listed for + all users. + + + ); + default: + return null; + } + }; + + const renderPrimaryAction = () => { + if (activeStep === 0) { + return ( + + ); + } + if (activeStep === 1) { + if (choice === SERVER) { + return ( + + ); + } + return ( + + ); + } + return ( + + ); + }; + + return ( + + Add dataset + + + {steps.map((label) => ( + + {label} + + ))} + + {getStepContent(activeStep)} + + + + + + {renderPrimaryAction()} + + + ); +}; + +AddDatasetDialog.propTypes = { + open: PropTypes.bool.isRequired, + onClose: PropTypes.func.isRequired, +}; + +export default AddDatasetDialog; diff --git a/frontend/src/pages/Datasets/DatasetDisplay.jsx b/frontend/src/pages/Datasets/DatasetDisplay.jsx index aec1059..f75a7fc 100644 --- a/frontend/src/pages/Datasets/DatasetDisplay.jsx +++ b/frontend/src/pages/Datasets/DatasetDisplay.jsx @@ -208,7 +208,7 @@ const DatasetDisplay = ({dataRecords, deleteMode}) => { {displayedData.map((record) => ( - {record.sample_sheet === "None" || record.sample_sheet === null || record.sample_sheet.trim() === "" ? record.dataset_id : + {record.is_remote || record.sample_sheet === "None" || record.sample_sheet === null || record.sample_sheet.trim() === "" ? (record.dataset_name || record.dataset_id) : {record.dataset_id}} {record.PI_full_name} @@ -218,10 +218,10 @@ const DatasetDisplay = ({dataRecords, deleteMode}) => { {record.assay} - {["scrnaseq", "snrnaseq", "visiumst","merfish"].includes(record.assay.toLowerCase()) && (UMAP)} - {/*{["visiumst","visium"].includes(record.assay.toLowerCase()) && (Visium)}*/} - {["eqtl", "caqtl"].includes(record.assay.toLowerCase()) && (xQTL)} - {record.has_bw && (Peaks)} + {["scrnaseq", "snrnaseq", "visiumst","merfish"].includes(record.assay.toLowerCase()) && (UMAP)} + {/*{["visiumst","visium"].includes(record.assay.toLowerCase()) && (Visium)}*/} + {["eqtl", "caqtl"].includes(record.assay.toLowerCase()) && (xQTL)} + {record.has_bw && (Peaks)} {deleteMode && ( @@ -242,7 +242,9 @@ const DatasetDisplay = ({dataRecords, deleteMode}) => { {displayedData.map((record) => ( - {record.dataset_id} + {record.is_remote || record.sample_sheet === "None" || record.sample_sheet === null || (record.sample_sheet ?? "").trim() === "" ? + (record.dataset_name || record.dataset_id) : + {record.dataset_id}} @@ -262,10 +264,10 @@ const DatasetDisplay = ({dataRecords, deleteMode}) => { - {["scrnaseq", "snrnaseq", "visiumst", "merfish"].includes(record.assay.toLowerCase()) && (UMAP)} - {/*{["visiumst","visium"].includes(record.assay.toLowerCase()) && (Visium)}*/} - {["eqtl", "caqtl"].includes(record.assay.toLowerCase()) && (xQTL)} - {record.has_bw && (Peaks)} + {["scrnaseq", "snrnaseq", "visiumst", "merfish"].includes(record.assay.toLowerCase()) && (UMAP)} + {/*{["visiumst","visium"].includes(record.assay.toLowerCase()) && (Visium)}*/} + {["eqtl", "caqtl"].includes(record.assay.toLowerCase()) && (xQTL)} + {record.has_bw && (Peaks)} {deleteMode && ( diff --git a/frontend/src/pages/Datasets/RemoteUrlForm.jsx b/frontend/src/pages/Datasets/RemoteUrlForm.jsx new file mode 100644 index 0000000..e707a43 --- /dev/null +++ b/frontend/src/pages/Datasets/RemoteUrlForm.jsx @@ -0,0 +1,203 @@ +import { forwardRef, useImperativeHandle, useState } from "react"; +import { + TextField, + Box, + Typography, + Chip, + IconButton, + List, + ListItem, + ListItemText, + Divider, + Link, + Alert, +} from "@mui/material"; +import DeleteIcon from "@mui/icons-material/Delete"; +import PropTypes from "prop-types"; +import { toast } from "react-toastify"; + +import useRemoteDatasetStore, { + buildRemoteRecord, +} from "../../store/RemoteDatasetStore.js"; +import useDatatableStore from "../../store/DatatableStore.js"; +import { inspectDataset } from "../../api/qtl.js"; + +const normalizeUrl = (raw) => (raw || "").trim().replace(/\/+$/, ""); + +const RemoteUrlForm = forwardRef(({ disabled, onBusyChange, onAdded }, ref) => { + const { remoteDatasets, addRemoteDataset, removeRemoteDataset } = + useRemoteDatasetStore(); + const { fetchDatasetList } = useDatatableStore(); + + const [url, setUrl] = useState(""); + const [name, setName] = useState(""); + const [error, setError] = useState(""); + + const setBusy = (busy) => { + if (onBusyChange) onBusyChange(busy); + }; + + const handleAdd = async () => { + setError(""); + const cleanUrl = normalizeUrl(url); + if (!/^https?:\/\//i.test(cleanUrl)) { + setError("Please enter a full http:// or https:// URL."); + return; + } + + setBusy(true); + try { + const info = await inspectDataset(cleanUrl); + if (!info || !info.reachable) { + setError( + info?.error || + "Could not reach a valid dataset at that URL. Check the URL, that the host is publicly reachable, and that it serves the dataset files (e.g. dataset_info.toml, celltypes/ or bigwig/).", + ); + setBusy(false); + return; + } + if (!info.has_qtl && !info.has_bw) { + setError( + "The URL is reachable but does not look like a QTL or signal/peaks dataset (no celltypes/ parquet and no bigwig/ tracks found).", + ); + setBusy(false); + return; + } + + const record = buildRemoteRecord(cleanUrl, name, info); + addRemoteDataset(record); + await fetchDatasetList(); + toast.success(`Added remote dataset "${record.dataset_name}".`); + setUrl(""); + setName(""); + setError(""); + setBusy(false); + if (onAdded) onAdded(record); + } catch (e) { + setError( + e?.response?.data?.detail || + "Failed to inspect the dataset URL. The server may have blocked it (private/loopback addresses are not allowed).", + ); + setBusy(false); + } + }; + + const handleRemove = async (datasetId) => { + removeRemoteDataset(datasetId); + await fetchDatasetList(); + toast.info("Removed remote dataset."); + }; + + useImperativeHandle(ref, () => ({ submit: handleAdd })); + + return ( + <> + + Point to the dataset directory of a remotely hosted HTTP server. + The dataset is stored only in your browser and is not visible to + other users. + + + + setUrl(e.target.value)} + fullWidth + size="small" + disabled={disabled} + autoFocus + /> + setName(e.target.value)} + fullWidth + size="small" + disabled={disabled} + /> + {error && {error}} + + + {remoteDatasets.length > 0 && ( + <> + + + Your remote datasets + + + {remoteDatasets.map((ds) => ( + + handleRemove(ds.dataset_id) + } + > + + + } + > + + {ds.dataset_name} + + {ds.has_bw && ( + + )} + + } + secondary={ + + {ds.dataset_id} + + } + /> + + ))} + + + )} + + ); +}); + +RemoteUrlForm.displayName = "RemoteUrlForm"; + +RemoteUrlForm.propTypes = { + disabled: PropTypes.bool, + onBusyChange: PropTypes.func, + onAdded: PropTypes.func, +}; + +export default RemoteUrlForm; diff --git a/frontend/src/pages/Datasets/index.jsx b/frontend/src/pages/Datasets/index.jsx index d2c10f3..5b428d6 100644 --- a/frontend/src/pages/Datasets/index.jsx +++ b/frontend/src/pages/Datasets/index.jsx @@ -1,15 +1,18 @@ import {useEffect, useState} from "react"; -import {Box, Typography, Divider, Button, Link} from "@mui/material"; +import {Box, Typography, Divider, Button} from "@mui/material"; import FilterPanel from "./FilterPanel"; import DatasetDisplay from "./DatasetDisplay.jsx"; +import AddDatasetDialog from "./AddDatasetDialog.jsx"; import "./DatasetPage.css"; import useDatatableStore from "../../store/DatatableStore.js"; +import useServerConfigStore from "../../store/ServerConfigStore.js"; import {useSearchParams} from "react-router-dom"; const DatasetsPage = () => { const {datasetRecords, fetchDatasetList} = useDatatableStore(); + const {fetchServerConfig} = useServerConfigStore(); const [searchParams, setSearchParams] = useSearchParams(); // Initialize filters from URL params @@ -26,6 +29,10 @@ const DatasetsPage = () => { fetchDatasetList() }, [fetchDatasetList]); + useEffect(() => { + fetchServerConfig() + }, [fetchServerConfig]); + // Update URL when filters change useEffect(() => { const newSearchParams = new URLSearchParams() @@ -123,6 +130,8 @@ const DatasetsPage = () => { setDeleteMode(!deleteMode); } + const [addDatasetOpen, setAddDatasetOpen] = useState(false); + return (
    {/* Title Row */} @@ -141,13 +150,15 @@ const DatasetsPage = () => { {/* {deleteMode ? "Cancel Delete" : "- Delete Dataset"}*/} {/**/} - + setAddDatasetOpen(false)}/> + {/* Main Content */} {/* Left Filter Panel */} diff --git a/frontend/src/pages/GenomicRegionView/index.jsx b/frontend/src/pages/GenomicRegionView/index.jsx index b93ea18..516ffce 100644 --- a/frontend/src/pages/GenomicRegionView/index.jsx +++ b/frontend/src/pages/GenomicRegionView/index.jsx @@ -33,6 +33,7 @@ import "./GenomicRegionView.css"; import useDataStore from "../../store/DatatableStore.js"; import useSignalStore from "../../store/GenomicRegionStore.js"; +import useRemoteDatasetStore from "../../store/RemoteDatasetStore.js"; import RegionViewPlotlyPlot from "./RegionViewPlotlyPlot.jsx"; @@ -77,15 +78,37 @@ function GenomicRegionView() { const urlRegion = queryParams.get("region") ?? ""; const { datasetRecords, fetchDatasetList } = useDataStore(); + const { ensureRemoteDataset } = useRemoteDatasetStore(); useEffect(() => { fetchDatasetList(); }, []); + // When a remote dataset is referenced by URL but not registered locally + // (e.g. when pasting in a shared link), register it so its display name is + // fetched from the dataset_info.toml + useEffect(() => { + if ( + urlDataset && + /^https?:\/\//i.test(urlDataset) && + !datasetRecords.some((d) => d.dataset_id === urlDataset) + ) { + ensureRemoteDataset(urlDataset).then((added) => { + if (added) fetchDatasetList(); + }); + } + }, [urlDataset, datasetRecords]); + const datasetOptions = datasetRecords .filter((d) => /[A-Za-z]*?(RNAseq|ATACseq)$/i.test(d.assay) && d.has_bw) .map((d) => d.dataset_id); + // Map dataset_id to display name + const datasetLabel = (id) => { + const ds = datasetRecords.find((d) => d.dataset_id === id); + return (ds && ds.dataset_name) || id || ""; + }; + const [datasetId, setDatasetId] = useState(urlDataset); const [datasetSearchText, setDatasetSearchText] = useState(""); @@ -786,6 +809,7 @@ function GenomicRegionView() { options={datasetOptions} value={datasetId ?? null} onChange={handleDatasetChange} + getOptionLabel={(option) => datasetLabel(option)} inputValue={datasetSearchText} onInputChange={(event, newInputValue) => setDatasetSearchText(newInputValue) @@ -802,7 +826,7 @@ function GenomicRegionView() { const { key, ...rest } = props; return (
  • - {option} + {datasetLabel(option)}
    • // // {option} diff --git a/frontend/src/pages/Help/HowToUse.jsx b/frontend/src/pages/Help/HowToUse.jsx index 79ac5e8..6991564 100644 --- a/frontend/src/pages/Help/HowToUse.jsx +++ b/frontend/src/pages/Help/HowToUse.jsx @@ -1017,7 +1017,16 @@ genes = [ "SNCA",...] ## List of default gene names 1. Upload the whole dataset folder to the server backend/datasets/ folder. The gene expression matrix file is not required(raw_normalized_counts.csv), Files with name starting with 'raw_' are not needed to be uploaded. 2. The sample sheet file is required(e.g., DatasetName_sample_sheet.csv) and must be uploaded to the backend/SampleSheets/ folder. 3. The dataset configuration file is required, the file name must be 'dataset_info.yaml' and must be put in the backend/datasets/<your_dataset_name>/ folder. - 4. Refresh the database: Dataset Manager + 4. Refresh the database on the server, from the repository root: python -m backend.db_utils.refresh_db + + Loading data from a URL (no backend access needed): + + 1. Available for xQTL and signal/peaks datasets. + 2. Serve the processed dataset folder from a web server that supports HTTP range requests, e.g. nginx, Apache, Caddy, or S3. Python's http.server will not work since it ignores Range headers. + 3. Make sure that URL is reachable from the machine running the VizIt backend. + 4. Go to DATASETS > + ADD DATASET and use the Load from URL option. + 5. The dataset is registered in your browser only. It is not visible to other users and needs no database refresh. + 6. Serving from a private or loopback address requires REMOTE_DATASET_ALLOW_PRIVATE=true in backend/.env. diff --git a/frontend/src/pages/Home_BDP/index.jsx b/frontend/src/pages/Home_BDP/index.jsx index ce020b1..323aece 100644 --- a/frontend/src/pages/Home_BDP/index.jsx +++ b/frontend/src/pages/Home_BDP/index.jsx @@ -111,7 +111,7 @@ const Home = () => { Loading stats data... } {/* Add Button here */} - diff --git a/frontend/src/pages/XQTLView/index.jsx b/frontend/src/pages/XQTLView/index.jsx index 6198e44..fa9a146 100644 --- a/frontend/src/pages/XQTLView/index.jsx +++ b/frontend/src/pages/XQTLView/index.jsx @@ -33,6 +33,7 @@ import "./XQTLView.css"; import useDataStore from "../../store/DatatableStore.js"; import useQtlStore from "../../store/QtlStore.js"; +import useRemoteDatasetStore from "../../store/RemoteDatasetStore.js"; import GeneViewPlotlyPlot from "./GeneViewPlotlyPlot.jsx"; import SNPViewPlotlyPlot from "./SNPViewPlotlyPlot.jsx"; @@ -79,14 +80,36 @@ function XQTLView() { const urlDataset = queryParams.get("dataset") ?? ""; const { datasetRecords, fetchDatasetList } = useDataStore(); + const { ensureRemoteDataset } = useRemoteDatasetStore(); useEffect(() => { fetchDatasetList(); }, []); + // When a remote dataset is referenced by URL but not registered locally + // (e.g. when pasting in a shared link), register it so its display name is + // fetched from the dataset_info.toml + useEffect(() => { + if ( + urlDataset && + /^https?:\/\//i.test(urlDataset) && + !datasetRecords.some((d) => d.dataset_id === urlDataset) + ) { + ensureRemoteDataset(urlDataset).then((added) => { + if (added) fetchDatasetList(); + }); + } + }, [urlDataset, datasetRecords]); + const datasetOptions = datasetRecords .filter((d) => d.assay.toLowerCase().endsWith("qtl")) .map((d) => d.dataset_id); + // Map dataset_id to display name + const datasetLabel = (id) => { + const ds = datasetRecords.find((d) => d.dataset_id === id); + return (ds && ds.dataset_name) || id || ""; + }; + const [datasetId, setDatasetId] = useState(urlDataset); const [datasetSearchText, setDatasetSearchText] = useState(""); @@ -551,6 +574,7 @@ function XQTLView() { options={datasetOptions} value={datasetId ?? null} onChange={handleDatasetChange} + getOptionLabel={(option) => datasetLabel(option)} inputValue={datasetSearchText} onInputChange={(event, newInputValue) => setDatasetSearchText(newInputValue) @@ -567,7 +591,7 @@ function XQTLView() { const { key, ...rest } = props; return (
  • - {option} + {datasetLabel(option)}
    • // // {option} diff --git a/frontend/src/store/DatasetManageStore.js b/frontend/src/store/DatasetManageStore.js index c1a7f6b..9c8b9c1 100644 --- a/frontend/src/store/DatasetManageStore.js +++ b/frontend/src/store/DatasetManageStore.js @@ -1,6 +1,6 @@ import {create} from 'zustand' import axios from 'axios' -import {toast} from "react-toastify"; +// import {toast} from "react-toastify"; const BASE_URL = import.meta.env.VITE_BACKEND_URL; const dmURL = `${BASE_URL}/datasetmanage`; @@ -194,21 +194,21 @@ const useDatasetManageStore = create((set, get) => ({ }); }, - refreshDatabase: async () => { - try { - const response = await axios.get(`${dmURL}/refreshdatabase`); - if(response.data.success){ - toast.success(response.data.message); - }else{ - toast.error(response.data.message); - } - return response.data; - } catch (error) { - console.error('Error refreshing database:', error); - toast.error('Error while refreshing database.'); - return null; - } - }, + // refreshDatabase: async () => { + // try { + // const response = await axios.get(`${dmURL}/refreshdatabase`); + // if(response.data.success){ + // toast.success(response.data.message); + // }else{ + // toast.error(response.data.message); + // } + // return response.data; + // } catch (error) { + // console.error('Error refreshing database:', error); + // toast.error('Error while refreshing database.'); + // return null; + // } + // }, deleteDataset: async (dataset) => { try { diff --git a/frontend/src/store/DatatableStore.js b/frontend/src/store/DatatableStore.js index 22b567a..c3f2523 100644 --- a/frontend/src/store/DatatableStore.js +++ b/frontend/src/store/DatatableStore.js @@ -2,6 +2,7 @@ import {create} from "zustand"; import {toast} from "react-toastify"; import {getDatatable_get, getDatasetList, getSampletable_get} from "../api/api.js"; import {checkBWDataExists} from "./GenomicRegionStore.js"; +import useRemoteDatasetStore from "./RemoteDatasetStore.js"; const useDatatableStore = create((set) => ({ dataRecords: [], @@ -61,6 +62,7 @@ const useDatatableStore = create((set) => ({ }, fetchDatasetList: async () => { + const remoteDatasets = useRemoteDatasetStore.getState().remoteDatasets || []; try { const response = await getDatasetList(); // console.log(response); @@ -75,21 +77,31 @@ const useDatatableStore = create((set) => ({ }) ); - await set({datasetRecords: recordsWithBW, datasetFilters: data[1], datasetfetchStatus: "success"}); + await set({datasetRecords: [...recordsWithBW, ...remoteDatasets], datasetFilters: data[1], datasetfetchStatus: "success"}); // toast.success("Sample loaded successfully!"); } else { console.error("Error fetching data:", response.data); - await set({datasetRecords: [], datasetfetchStatus: "failed"}); + await set({datasetRecords: [...remoteDatasets], datasetfetchStatus: "failed"}); toast.error("Failed to fetch datasets."); } } catch (error) { console.error("Error fetching data:", error); - await set({datasetRecords: [], datasetfetchStatus: "error"}); - toast.info(error.response.data.detail); + // Use browser-local remote datasets if the server dataset table is unavailable. + await set({datasetRecords: [...remoteDatasets], datasetfetchStatus: "error"}); + if (error?.response?.data?.detail) { + toast.info(error.response.data.detail); + } } }, })); +// When remote datasets change, refresh the merged list so all views pick up the +// updated remote datasets. +useRemoteDatasetStore.subscribe( + (state) => state.remoteDatasets, + () => useDatatableStore.getState().fetchDatasetList(), +); + export default useDatatableStore; diff --git a/frontend/src/store/RemoteDatasetStore.js b/frontend/src/store/RemoteDatasetStore.js new file mode 100644 index 0000000..0f3f76b --- /dev/null +++ b/frontend/src/store/RemoteDatasetStore.js @@ -0,0 +1,89 @@ +import { create } from "zustand"; +import { persist } from "zustand/middleware"; + +import { inspectDataset } from "../api/qtl.js"; + +export const buildRemoteRecord = (url, name, info = {}) => { + let assay = info.assay || ""; + const hasBw = !!info.has_bw; + const hasQtl = !!info.has_qtl; + + // Normalize the assay string so the view filters recognize it: + // - xQTL view lists datasets whose assay ends with "qtl" + // - Genomic-Region view lists datasets whose assay ends with RNAseq/ATACseq + if (hasQtl && !assay.toLowerCase().endsWith("qtl")) { + assay = info.is_caqtl ? "caQTL" : "eQTL"; + } + if (hasBw && !/(RNAseq|ATACseq)$/i.test(assay)) { + assay = + assay && assay.toLowerCase().endsWith("qtl") ? assay : "scATACseq"; + } + + return { + dataset_id: url, + dataset_name: name || info.dataset_name || url, + PI_full_name: (info.info && info.info.PI_full_name) || "—", + first_contributor: (info.info && info.info.first_contributor) || "—", + n_samples: (info.info && info.info.n_samples) || 0, + brain_region: (info.info && info.info.brain_region) || "", + brain_super_region: (info.info && info.info.brain_super_region) || "", + disease: (info.info && info.info.disease) || "", + organism: (info.info && info.info.organism) || "", + tissue: (info.info && info.info.tissue) || "", + assay: assay, + sample_sheet: "None", + has_bw: hasBw, + is_remote: true, + }; +}; + +const useRemoteDatasetStore = create( + persist( + (set, get) => ({ + remoteDatasets: [], + + addRemoteDataset: (record) => + set((state) => ({ + remoteDatasets: [ + ...state.remoteDatasets.filter( + (r) => r.dataset_id !== record.dataset_id, + ), + record, + ], + })), + + ensureRemoteDataset: async (url) => { + if (!url || !/^https?:\/\//i.test(url)) return false; + if (get().remoteDatasets.some((r) => r.dataset_id === url)) { + return false; + } + try { + const info = await inspectDataset(url); + if (!info || !info.reachable) return false; + const record = buildRemoteRecord(url, "", info); + set((state) => ({ + remoteDatasets: [ + ...state.remoteDatasets.filter( + (r) => r.dataset_id !== url, + ), + record, + ], + })); + return true; + } catch { + return false; + } + }, + + removeRemoteDataset: (datasetId) => + set((state) => ({ + remoteDatasets: state.remoteDatasets.filter( + (r) => r.dataset_id !== datasetId, + ), + })), + }), + { name: "vizit-remote-datasets" }, + ), +); + +export default useRemoteDatasetStore; diff --git a/frontend/src/store/ServerConfigStore.js b/frontend/src/store/ServerConfigStore.js new file mode 100644 index 0000000..d8accc9 --- /dev/null +++ b/frontend/src/store/ServerConfigStore.js @@ -0,0 +1,29 @@ +import { create } from "zustand"; + +import { getServerConfig } from "../api/api.js"; + +// `allowRemoteDatasets` starts as null so that an older backend without a +// /serverconfig endpoint keeps the remote-dataset UI visible (backward +// compatible). The UI hides the feature only when the backend explicitly +// reports it as disabled. +const useServerConfigStore = create((set, get) => ({ + allowRemoteDatasets: null, + remoteCapabilities: [], + loaded: false, + + fetchServerConfig: async () => { + if (get().loaded) return; + try { + const cfg = await getServerConfig(); + set({ + allowRemoteDatasets: !!cfg.allow_remote_datasets, + remoteCapabilities: cfg.remote_capabilities || [], + loaded: true, + }); + } catch { + set({ loaded: true }); + } + }, +})); + +export default useServerConfigStore;