From 963a2cba41a183fd6b71c0dffcca73e52c7e02c3 Mon Sep 17 00:00:00 2001 From: eric Date: Wed, 1 Jul 2026 13:41:45 -0500 Subject: [PATCH 1/6] add index_file guppy index --- .../gen3.biodatacatalyst.nhlbi.nih.gov/values/values.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bdcatprod/gen3.biodatacatalyst.nhlbi.nih.gov/values/values.yaml b/bdcatprod/gen3.biodatacatalyst.nhlbi.nih.gov/values/values.yaml index f36f78a74..a5ad5146d 100644 --- a/bdcatprod/gen3.biodatacatalyst.nhlbi.nih.gov/values/values.yaml +++ b/bdcatprod/gen3.biodatacatalyst.nhlbi.nih.gov/values/values.yaml @@ -3213,7 +3213,7 @@ fence: region: us-east-1 nih-nhlbi-topmed-sumstats-phs001974-r1-v9-c1: cred: fence-bot - region: us-east-1 + region: us-east-1 ENABLE_VISA_UPDATE_CRON: true fluentd: enabled: true @@ -3285,6 +3285,8 @@ guppy: type: subject - index: gen3.biodatacatalyst.nhlbi.nih.gov_file type: file + - index: gen3.biodatacatalyst.nhlbi.nih.gov_index_file + type: index_file hatchery: enabled: false indexd: From beb7304dfbfae09557964524949046db3ffd9a33 Mon Sep 17 00:00:00 2001 From: eric Date: Wed, 1 Jul 2026 13:52:34 -0500 Subject: [PATCH 2/6] reorg etlMappings --- .../values/values.yaml | 68 +++++++++---------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/bdcatprod/gen3.biodatacatalyst.nhlbi.nih.gov/values/values.yaml b/bdcatprod/gen3.biodatacatalyst.nhlbi.nih.gov/values/values.yaml index a5ad5146d..a05f6231f 100644 --- a/bdcatprod/gen3.biodatacatalyst.nhlbi.nih.gov/values/values.yaml +++ b/bdcatprod/gen3.biodatacatalyst.nhlbi.nih.gov/values/values.yaml @@ -54,7 +54,16 @@ etl: esEndpoint: elasticsearch etlMapping: mappings: - - aggregated_props: + - name: gen3.biodatacatalyst.nhlbi.nih.gov_subject + doc_type: subject + type: aggregator + root: subject + props: + - name: submitter_id + - name: project_id + - name: consent_codes + - name: geographic_site + aggregated_props: - fn: set name: annotated_sex path: demographics @@ -93,7 +102,9 @@ etl: name: ever_smoker_baseline path: exposures src: ever_smoker_baseline - doc_type: subject + parent_props: + - path: studies[studies_submitter_id:submitter_id].projects[code] + - path: studies[studies_submitter_id:submitter_id].projects[code].programs[programs_name:name] flatten_props: - path: demographics props: @@ -167,19 +178,23 @@ etl: - fn: count name: file_count src: _file_id - name: gen3.biodatacatalyst.nhlbi.nih.gov_subject - parent_props: - - path: studies[studies_submitter_id:submitter_id].projects[code] - - path: studies[studies_submitter_id:submitter_id].projects[code].programs[programs_name:name] - props: - - name: submitter_id - - name: project_id - - name: consent_codes - - name: geographic_site - root: subject - type: aggregator - - category: data_file + - name: gen3.biodatacatalyst.nhlbi.nih.gov_file doc_type: file + type: collector + root: None + category: data_file + props: + - name: object_id + - name: md5sum + - name: file_name + - name: file_size + - name: data_category + - name: data_format + - name: data_type + - name: state + - name: callset + - name: bucket_path + - name: source_node injecting_props: program: props: @@ -197,7 +212,11 @@ etl: name: _subject_id src: id - name: project_id - name: gen3.biodatacatalyst.nhlbi.nih.gov_file + - name: gen3.biodatacatalyst.nhlbi.nih.gov_index_file + doc_type: index_file + root: None + type: collector + category: index_file props: - name: object_id - name: md5sum @@ -210,10 +229,6 @@ etl: - name: callset - name: bucket_path - name: source_node - root: None - type: collector - - category: index_file - doc_type: index_file injecting_props: program: props: @@ -231,21 +246,6 @@ etl: name: _subject_id src: id - name: project_id - name: gen3.biodatacatalyst.nhlbi.nih.gov_index_file - props: - - name: object_id - - name: md5sum - - name: file_name - - name: file_size - - name: data_category - - name: data_format - - name: data_type - - name: state - - name: callset - - name: bucket_path - - name: source_node - root: None - type: collector image: tube: tag: '2026.04' From 08ca819448a116a46db4984a159babcd93efe1c4 Mon Sep 17 00:00:00 2001 From: eric Date: Wed, 1 Jul 2026 13:56:48 -0500 Subject: [PATCH 3/6] add index_file to subject joining_props --- .../values/values.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/bdcatprod/gen3.biodatacatalyst.nhlbi.nih.gov/values/values.yaml b/bdcatprod/gen3.biodatacatalyst.nhlbi.nih.gov/values/values.yaml index a05f6231f..34294c489 100644 --- a/bdcatprod/gen3.biodatacatalyst.nhlbi.nih.gov/values/values.yaml +++ b/bdcatprod/gen3.biodatacatalyst.nhlbi.nih.gov/values/values.yaml @@ -178,6 +178,18 @@ etl: - fn: count name: file_count src: _file_id + - index: index_file + join_on: _subject_id + props: + - fn: set + name: data_format + src: data_format + - fn: set + name: data_type + src: data_type + - fn: count + name: file_count + src: _file_id - name: gen3.biodatacatalyst.nhlbi.nih.gov_file doc_type: file type: collector From ec86452d9805e3e88087ac91e511e88f0d7427eb Mon Sep 17 00:00:00 2001 From: eric <98181083+ciresiemanym@users.noreply.github.com> Date: Wed, 1 Jul 2026 15:59:27 -0500 Subject: [PATCH 4/6] further reorg --- .../values/values.yaml | 104 +++++++++--------- 1 file changed, 55 insertions(+), 49 deletions(-) diff --git a/bdcatprod/gen3.biodatacatalyst.nhlbi.nih.gov/values/values.yaml b/bdcatprod/gen3.biodatacatalyst.nhlbi.nih.gov/values/values.yaml index 34294c489..86237dc6a 100644 --- a/bdcatprod/gen3.biodatacatalyst.nhlbi.nih.gov/values/values.yaml +++ b/bdcatprod/gen3.biodatacatalyst.nhlbi.nih.gov/values/values.yaml @@ -64,44 +64,44 @@ etl: - name: consent_codes - name: geographic_site aggregated_props: - - fn: set - name: annotated_sex + - name: annotated_sex path: demographics src: annotated_sex - - fn: set - name: race + fn: set + - name: race path: demographics src: race - - fn: set - name: ethnicity + fn: set + - name: ethnicity path: demographics src: ethnicity - - fn: count - name: _samples_count + fn: set + - name: _samples_count path: samples - - fn: count - name: _aliquots_count + fn: count + - name: _aliquots_count path: samples.aliquots - - fn: count - name: _read_groups_count + fn: count + - name: _read_groups_count path: samples.aliquots.read_groups - - fn: count - name: _submitted_unaligned_reads_files_count + fn: count + - name: _submitted_unaligned_reads_files_count path: samples.aliquots.read_groups.submitted_unaligned_reads_files - - fn: count - name: _submitted_aligned_reads_files_count + fn: count + - name: _submitted_aligned_reads_files_count path: samples.aliquots.read_groups.submitted_aligned_reads_files - - fn: count - name: _simple_germline_variations_count + fn: count + - name: _simple_germline_variations_count path: samples.aliquots.read_groups.simple_germline_variations - - fn: set - name: current_smoker_baseline + fn: count + - name: current_smoker_baseline path: exposures src: current_smoker_baseline - - fn: set - name: ever_smoker_baseline + fn: set + - name: ever_smoker_baseline path: exposures src: ever_smoker_baseline + fn: set parent_props: - path: studies[studies_submitter_id:submitter_id].projects[code] - path: studies[studies_submitter_id:submitter_id].projects[code].programs[programs_name:name] @@ -169,33 +169,40 @@ etl: - index: file join_on: _subject_id props: - - fn: set - name: data_format + - name: object_id + src: object_id + fn: set + - name: data_format src: data_format - - fn: set - name: data_type + fn: set + - name: data_type src: data_type - - fn: count - name: file_count + fn: set + - name: file_count src: _file_id + fn: count - index: index_file join_on: _subject_id props: - - fn: set - name: data_format + - name: object_id + src: object_id + fn: set + - name: data_format src: data_format - - fn: set - name: data_type + fn: set + - name: data_type src: data_type - - fn: count - name: file_count + fn: set + - name: file_count src: _file_id + fn: count - name: gen3.biodatacatalyst.nhlbi.nih.gov_file doc_type: file type: collector root: None category: data_file props: + - name: project_id - name: object_id - name: md5sum - name: file_name @@ -210,26 +217,26 @@ etl: injecting_props: program: props: - - fn: set - name: programs_name + - name: programs_name src: name + fn: set project: props: - - fn: set - name: dbgap_accession_number + - name: dbgap_accession_number src: dbgap_accession_number + fn: set subject: props: - - fn: set - name: _subject_id + - name: _subject_id src: id - - name: project_id + fn: set - name: gen3.biodatacatalyst.nhlbi.nih.gov_index_file doc_type: index_file - root: None type: collector + root: None category: index_file props: + - name: project_id - name: object_id - name: md5sum - name: file_name @@ -244,20 +251,19 @@ etl: injecting_props: program: props: - - fn: set - name: programs_name + - name: programs_name src: name + fn: set project: props: - - fn: set - name: dbgap_accession_number + - name: dbgap_accession_number src: dbgap_accession_number + fn: set subject: props: - - fn: set - name: _subject_id + - name: _subject_id src: id - - name: project_id + fn: set image: tube: tag: '2026.04' From 27cc69e19e8cd273babee0fde381f71b956ed426 Mon Sep 17 00:00:00 2001 From: eric <98181083+ciresiemanym@users.noreply.github.com> Date: Thu, 2 Jul 2026 08:45:18 -0500 Subject: [PATCH 5/6] use index_file_count for the index_file subject joining_prop, update count prop to index_file_count --- .../gen3.biodatacatalyst.nhlbi.nih.gov/values/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bdcatprod/gen3.biodatacatalyst.nhlbi.nih.gov/values/values.yaml b/bdcatprod/gen3.biodatacatalyst.nhlbi.nih.gov/values/values.yaml index 86237dc6a..78e96cc6f 100644 --- a/bdcatprod/gen3.biodatacatalyst.nhlbi.nih.gov/values/values.yaml +++ b/bdcatprod/gen3.biodatacatalyst.nhlbi.nih.gov/values/values.yaml @@ -193,8 +193,8 @@ etl: - name: data_type src: data_type fn: set - - name: file_count - src: _file_id + - name: index_file_count + src: _index_file_id fn: count - name: gen3.biodatacatalyst.nhlbi.nih.gov_file doc_type: file From 778b715da3fb0bbafe7eedfacef6021948623501 Mon Sep 17 00:00:00 2001 From: eric <98181083+ciresiemanym@users.noreply.github.com> Date: Thu, 2 Jul 2026 12:27:01 -0500 Subject: [PATCH 6/6] make file and index_file joining_props names unique --- .../values/values.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bdcatprod/gen3.biodatacatalyst.nhlbi.nih.gov/values/values.yaml b/bdcatprod/gen3.biodatacatalyst.nhlbi.nih.gov/values/values.yaml index 78e96cc6f..3029da731 100644 --- a/bdcatprod/gen3.biodatacatalyst.nhlbi.nih.gov/values/values.yaml +++ b/bdcatprod/gen3.biodatacatalyst.nhlbi.nih.gov/values/values.yaml @@ -169,13 +169,13 @@ etl: - index: file join_on: _subject_id props: - - name: object_id + - name: file_object_id src: object_id fn: set - - name: data_format + - name: file_data_format src: data_format fn: set - - name: data_type + - name: file_data_type src: data_type fn: set - name: file_count @@ -184,13 +184,13 @@ etl: - index: index_file join_on: _subject_id props: - - name: object_id + - name: index_file_object_id src: object_id fn: set - - name: data_format + - name: index_file_data_format src: data_format fn: set - - name: data_type + - name: index_file_data_type src: data_type fn: set - name: index_file_count