From b2ac178d7419d6d0f99a6a61d2e49c49961788d3 Mon Sep 17 00:00:00 2001 From: Rendani Gangazhe Date: Fri, 2 Feb 2024 01:14:38 +0100 Subject: [PATCH 01/34] Implement history carbon emissions reporting (WIP) --- client/src/api/index.ts | 10 + client/src/api/schema/schema.ts | 127 +++++++++++ .../CarbonEmissions/BarChart.vue | 0 .../CarbonEmissions/CarbonEmissionCard.vue | 0 .../CarbonEmissions/CarbonEmissions.test.js | 0 .../CarbonEmissions/CarbonEmissions.vue | 84 +++++-- .../CarbonEmissionsCalculations.vue | 0 .../CarbonEmissions/CarbonEmissionsIcon.vue | 0 .../carbonEmissionConstants.js | 0 .../History/CurrentHistory/HistoryCounter.vue | 56 +++-- .../History/CurrentHistory/HistoryMetrics.vue | 89 ++++++++ client/src/components/History/HistoryView.vue | 213 ++++++++++-------- .../src/components/JobMetrics/JobMetrics.vue | 59 +---- client/src/composables/carbonEmissions.ts | 49 ++++ client/src/entry/analysis/router.js | 9 +- lib/galaxy/managers/jobs.py | 4 +- .../webapps/galaxy/api/history_contents.py | 24 ++ lib/galaxy/webapps/galaxy/buildapp.py | 1 + .../galaxy/services/history_contents.py | 117 ++++++++++ 19 files changed, 672 insertions(+), 170 deletions(-) rename client/src/components/{JobMetrics => }/CarbonEmissions/BarChart.vue (100%) rename client/src/components/{JobMetrics => }/CarbonEmissions/CarbonEmissionCard.vue (100%) rename client/src/components/{JobMetrics => }/CarbonEmissions/CarbonEmissions.test.js (100%) rename client/src/components/{JobMetrics => }/CarbonEmissions/CarbonEmissions.vue (87%) rename client/src/components/{JobMetrics => }/CarbonEmissions/CarbonEmissionsCalculations.vue (100%) rename client/src/components/{JobMetrics => }/CarbonEmissions/CarbonEmissionsIcon.vue (100%) rename client/src/components/{JobMetrics => }/CarbonEmissions/carbonEmissionConstants.js (100%) create mode 100644 client/src/components/History/CurrentHistory/HistoryMetrics.vue create mode 100644 client/src/composables/carbonEmissions.ts diff --git a/client/src/api/index.ts b/client/src/api/index.ts index 0003833819fa..b646708d79a9 100644 --- a/client/src/api/index.ts +++ b/client/src/api/index.ts @@ -17,6 +17,16 @@ export type HistoryDetailed = components["schemas"]["HistoryDetailed"]; */ export type HistoryContentItemBase = components["schemas"]["EncodedHistoryContentItem"]; +/** + * Contains information about the aggregated metrics of all jobs in a history. + */ +export type HistoryMetrics = components["schemas"]["HistoryMetrics"]; + +/** + * Contains information about the carbon emissions of a history. + */ +export type HistoryEmissions = components["schemas"]["HistoryEmissions"]; + /** * Contains summary information about a HistoryDatasetAssociation. */ diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 3d737e261904..de5bd1a9b563 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -537,6 +537,14 @@ export interface paths { /** Marks the history with the given ID as deleted. */ delete: operations["delete_api_histories__history_id__delete"]; }; + "/api/histories/{history_id}/metrics": { + /** Returns the cumulative metrics for all jobs in a given history with ``history_id``. */ + get: operations["history_api_histories__history_id__metrics_get"]; + }; + "/api/histories/{history_id}/emissions": { + /** Returns the cumulative metrics for all jobs in a given history with ``history_id``. */ + get: operations["history_api_histories__history_id__emissions_get"]; + }; "/api/histories/{history_id}/archive": { /** * Archive a history. @@ -6534,6 +6542,73 @@ export interface components { url: string; [key: string]: unknown | undefined; }; + /** + * History Metrics + * @description The accumulated metrics of all jobs in a history. + */ + HistoryMetrics: { + /** + Total Jobs Count + @description The total number of jobs in the history. + */ + total_jobs_in_history: number; + /** + Total Runtime in Seconds + @description The total amount of compute runtime used by all jobs in the history. + */ + total_runtime_in_seconds: number; + /** + Total Cores Allocated + @description The total numbers of cores allocated for all jobs in a history. + */ + total_cores_allocated: number; + /** + Total Memory Allocated + @description The total amount of memory used and alloceated for all jobs in a history. + */ + total_memory_allocated_in_mebibyte: number; + }; + /** + * History Emissions + * @description The carbon emissions of a history. + */ + HistoryEmissions: { + /** + NAME + @description DESC + */ + cpu_carbon_emissions: number; + + /** + NAME + @description DESC + */ + memory_carbon_emissions: number; + + /** + NAME + @description DESC + */ + total_carbon_emissions: number; + + /** + NAME + @description DESC + */ + energy_needed_cpu: number; + + /** + NAME + @description DESC + */ + energy_needed_memory: number; + + /** + NAME + @description DESC + */ + total_energy_needed: number; + }; /** * Hyperlink * @description Represents some text with an Hyperlink. @@ -14407,6 +14482,58 @@ export interface operations { }; }; }; + history_api_histories__history_id__metrics_get: { + parameters: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string | null; + }; + /** @description The encoded database identifier of the History. */ + path: { + history_id: string; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": components["schemas"]["HistoryMetrics"]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + history_api_histories__history_id__emissions_get: { + parameters: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string | null; + }; + /** @description The encoded database identifier of the History. */ + path: { + history_id: string; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": components["schemas"]["HistoryEmissions"]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; archive_history_api_histories__history_id__archive_post: { /** * Archive a history. diff --git a/client/src/components/JobMetrics/CarbonEmissions/BarChart.vue b/client/src/components/CarbonEmissions/BarChart.vue similarity index 100% rename from client/src/components/JobMetrics/CarbonEmissions/BarChart.vue rename to client/src/components/CarbonEmissions/BarChart.vue diff --git a/client/src/components/JobMetrics/CarbonEmissions/CarbonEmissionCard.vue b/client/src/components/CarbonEmissions/CarbonEmissionCard.vue similarity index 100% rename from client/src/components/JobMetrics/CarbonEmissions/CarbonEmissionCard.vue rename to client/src/components/CarbonEmissions/CarbonEmissionCard.vue diff --git a/client/src/components/JobMetrics/CarbonEmissions/CarbonEmissions.test.js b/client/src/components/CarbonEmissions/CarbonEmissions.test.js similarity index 100% rename from client/src/components/JobMetrics/CarbonEmissions/CarbonEmissions.test.js rename to client/src/components/CarbonEmissions/CarbonEmissions.test.js diff --git a/client/src/components/JobMetrics/CarbonEmissions/CarbonEmissions.vue b/client/src/components/CarbonEmissions/CarbonEmissions.vue similarity index 87% rename from client/src/components/JobMetrics/CarbonEmissions/CarbonEmissions.vue rename to client/src/components/CarbonEmissions/CarbonEmissions.vue index f124643cecbc..133d78d572de 100644 --- a/client/src/components/JobMetrics/CarbonEmissions/CarbonEmissions.vue +++ b/client/src/components/CarbonEmissions/CarbonEmissions.vue @@ -3,7 +3,7 @@ import { library } from "@fortawesome/fontawesome-svg-core"; import { faQuestionCircle } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome"; import type { GetComponentPropTypes } from "types/utilityTypes"; -import { computed, unref } from "vue"; +import { computed, ref, unref } from "vue"; import * as carbonEmissionsConstants from "./carbonEmissionConstants.js"; @@ -14,32 +14,32 @@ import Heading from "@/components/Common/Heading.vue"; library.add(faQuestionCircle); interface CarbonEmissionsProps { - estimatedServerInstance: { - name: string; - cpuInfo: { - modelName: string; - totalAvailableCores: number; - tdp: number; - }; - }; jobRuntimeInSeconds: number; coresAllocated: number; powerUsageEffectiveness: number; geographicalServerLocationName: string; memoryAllocatedInMebibyte?: number; carbonIntensity: number; + showHeader: boolean; + showCarbonEmissionsCalculationsLink: boolean; } const props = withDefaults(defineProps(), { memoryAllocatedInMebibyte: 0, + showHeader: false, + showCarbonEmissionsCalculationsLink: false, }); const carbonEmissions = computed(() => { + if (!estimatedServerInstance.value) { + return; + } + const memoryPowerUsed = carbonEmissionsConstants.memoryPowerUsage; const runtimeInHours = props.jobRuntimeInSeconds / (60 * 60); // Convert to hours const memoryAllocatedInGibibyte = props.memoryAllocatedInMebibyte / 1024; // Convert to gibibyte - const cpuInfo = props.estimatedServerInstance.cpuInfo; + const cpuInfo = estimatedServerInstance.value.cpuInfo; const tdpPerCore = cpuInfo.tdp / cpuInfo.totalAvailableCores; const normalizedTdpPerCore = tdpPerCore * props.coresAllocated; @@ -193,6 +193,45 @@ const barChartData = computed(() => { return data; }); +const estimatedServerInstance = computed(() => { + const ec2 = unref(ec2Instances); + if (!ec2) { + return; + } + + const memory = props.memoryAllocatedInMebibyte; + const adjustedMemory = memory ? memory / 1024 : 0; + const cores = props.coresAllocated; + + const serverInstance = ec2.find((instance) => { + if (adjustedMemory === 0) { + // Exclude memory from search criteria + return instance.vCpuCount >= cores; + } + + // Search by all criteria + return instance.mem >= adjustedMemory && instance.vCpuCount >= cores; + }); + + if (!serverInstance) { + return; + } + + const cpu = serverInstance.cpu[0]; + if (!cpu) { + return; + } + + return { + name: serverInstance.name, + cpuInfo: { + modelName: cpu.cpuModel, + totalAvailableCores: cpu.coreCount, + tdp: cpu.tdp, + }, + }; +}); + function prettyPrintValue({ value, threshold, @@ -266,11 +305,28 @@ function getEnergyNeededText(energyNeededInKiloWattHours: number) { const roundedValue = Math.round(adjustedEnergyNeeded); return `${roundedValue === 0 ? "< 1" : roundedValue} ${unitMagnitude}`; } + +const ec2Instances = ref(); +import("@/components/JobMetrics/awsEc2ReferenceData.js").then((data) => (ec2Instances.value = data.ec2Instances)); + +type EC2 = { + name: string; + mem: number; + price: number; + priceUnit: string; + vCpuCount: number; + cpu: { + cpuModel: string; + tdp: number; + coreCount: number; + source: string; + }[]; +}; - diff --git a/client/src/entry/analysis/router.js b/client/src/entry/analysis/router.js index d21dc1fa577d..2a5f20d0b1f9 100644 --- a/client/src/entry/analysis/router.js +++ b/client/src/entry/analysis/router.js @@ -13,7 +13,6 @@ import historiesSharedGridConfig from "components/Grid/configs/historiesShared"; import visualizationsGridConfig from "components/Grid/configs/visualizations"; import visualizationsPublishedGridConfig from "components/Grid/configs/visualizationsPublished"; import GridList from "components/Grid/GridList"; -import HistoryMetrics from "components/History/CurrentHistory/HistoryMetrics"; import HistoryExportTasks from "components/History/Export/HistoryExport"; import HistoryPublished from "components/History/HistoryPublished"; import HistoryPublishedList from "components/History/HistoryPublishedList"; @@ -70,9 +69,9 @@ import { patchRouterPush } from "./router-push"; import AboutGalaxy from "@/components/AboutGalaxy.vue"; import HistoryArchive from "@/components/History/Archiving/HistoryArchive.vue"; import HistoryArchiveWizard from "@/components/History/Archiving/HistoryArchiveWizard.vue"; +import HistoryStatistics from "@/components/History/Statistics/HistoryStatistics.vue"; import NotificationsList from "@/components/Notifications/NotificationsList.vue"; import Sharing from "@/components/Sharing/SharingPage.vue"; -import HistoryStorageOverview from "@/components/User/DiskUsage/Visualizations/HistoryStorageOverview.vue"; import WorkflowPublished from "@/components/Workflow/Published/WorkflowPublished.vue"; Vue.use(VueRouter); @@ -326,12 +325,6 @@ export function getRouter(Galaxy) { component: HistoryInvocations, props: true, }, - { - path: "histories/:historyId/metrics", - name: "HistoryMetrics", - component: HistoryMetrics, - props: true, - }, { path: "interactivetool_entry_points/list", component: InteractiveTools, @@ -384,13 +377,13 @@ export function getRouter(Galaxy) { path: "pages/:actionId", component: PageList, props: (route) => ({ - published: route.params.actionId == "list_published" ? true : false, + published: route.params.actionId === "list_published" ? true : false, }), }, { - path: "storage/history/:historyId", - name: "HistoryOverviewInAnalysis", - component: HistoryStorageOverview, + path: "statistics/history/:historyId", + name: "HistoryStatistics", + component: HistoryStatistics, props: true, }, { @@ -535,7 +528,7 @@ export function getRouter(Galaxy) { { path: "workflows/list_published", component: WorkflowList, - props: (route) => ({ + props: (_) => ({ published: true, }), }, @@ -544,9 +537,9 @@ export function getRouter(Galaxy) { component: WorkflowList, redirect: redirectAnon(), props: (route) => ({ - importMessage: route.query["message"], - importStatus: route.query["status"], - query: route.query["query"], + importMessage: route.query.message, + importStatus: route.query.status, + query: route.query.query, }), }, { @@ -574,7 +567,7 @@ export function getRouter(Galaxy) { queryTrsId: route.query.trs_id, queryTrsVersionId: route.query.trs_version, queryTrsUrl: route.query.trs_url, - isRun: route.query.run_form == "true", + isRun: route.query.run_form === "true", }), }, { @@ -611,7 +604,7 @@ export function getRouter(Galaxy) { return false; } - router.beforeEach(async (to, from, next) => { + router.beforeEach(async (to, _, next) => { // TODO: merge anon redirect functionality here for more standard handling const isAdminAccessRequired = checkAdminAccessRequired(to); diff --git a/lib/galaxy/job_metrics/instrumenters/core.py b/lib/galaxy/job_metrics/instrumenters/core.py index f449c5b29a9f..acf1a4e59fa1 100644 --- a/lib/galaxy/job_metrics/instrumenters/core.py +++ b/lib/galaxy/job_metrics/instrumenters/core.py @@ -23,21 +23,49 @@ END_EPOCH_KEY = "end_epoch" RUNTIME_SECONDS_KEY = "runtime_seconds" +ENERGY_NEEDED_MEMORY_KEY = "energy_needed_memory" +ENERGY_NEEDED_CPU_KEY = "energy_needed_cpu" + class CorePluginFormatter(JobMetricFormatter): def format(self, key: str, value: Any) -> FormattedMetric: - value = int(value) if key == GALAXY_SLOTS_KEY: - return FormattedMetric("Cores Allocated", "%d" % value) + return FormattedMetric("Cores Allocated", "%d" % int(value)) elif key == GALAXY_MEMORY_MB_KEY: - return FormattedMetric("Memory Allocated (MB)", "%d" % value) + return FormattedMetric("Memory Allocated (MB)", "%d" % int(value)) elif key == RUNTIME_SECONDS_KEY: - return FormattedMetric("Job Runtime (Wall Clock)", seconds_to_str(value)) + return FormattedMetric("Job Runtime (Wall Clock)", seconds_to_str(int(value))) + elif key == ENERGY_NEEDED_CPU_KEY: + return FormattedMetric("CPU Energy Usage", self.__format_energy_needed_text(float(value))) + elif key == ENERGY_NEEDED_MEMORY_KEY: + return FormattedMetric("Memory Energy Usage", self.__format_energy_needed_text(float(value))) else: # TODO: Use localized version of this from galaxy.ini title = "Job Start Time" if key == START_EPOCH_KEY else "Job End Time" return FormattedMetric(title, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(value))) + def __format_energy_needed_text(self, energy_needed_kwh: float) -> str: + adjustedEnergyNeeded = energy_needed_kwh + unit_magnitude = "kW⋅h" + + if energy_needed_kwh == 0: + return "0 kW⋅h" + + if energy_needed_kwh >= 1e3: + adjustedEnergyNeeded /= 1000 + unit_magnitude = "MW⋅h" + elif energy_needed_kwh >= 1 and energy_needed_kwh <= 999: + unit_magnitude = "W⋅h" + elif energy_needed_kwh < 1 and energy_needed_kwh > 1e-4: + adjustedEnergyNeeded *= 1000 + unit_magnitude = "mW⋅h" + else: + adjustedEnergyNeeded *= 1e6 + unit_magnitude = "µW⋅h" + + rounded_value = round(adjustedEnergyNeeded) == 0 and "< 1" or round(adjustedEnergyNeeded) + return f"{rounded_value} {unit_magnitude}" + class CorePlugin(InstrumentPlugin): """Simple plugin that collects data without external dependencies. In @@ -67,15 +95,48 @@ def job_properties(self, job_id, job_directory: str) -> Dict[str, Any]: galaxy_slots_file = self.__galaxy_slots_file(job_directory) galaxy_memory_mb_file = self.__galaxy_memory_mb_file(job_directory) - properties = {} - properties[GALAXY_SLOTS_KEY] = self.__read_integer(galaxy_slots_file) - properties[GALAXY_MEMORY_MB_KEY] = self.__read_integer(galaxy_memory_mb_file) - start = self.__read_seconds_since_epoch(job_directory, "start") - end = self.__read_seconds_since_epoch(job_directory, "end") - if start is not None and end is not None: - properties[START_EPOCH_KEY] = start - properties[END_EPOCH_KEY] = end - properties[RUNTIME_SECONDS_KEY] = end - start + properties: Dict[str, Any] = {} + + allocated_cpu_cores = self.__read_integer(galaxy_slots_file) + properties[GALAXY_SLOTS_KEY] = allocated_cpu_cores + + allocated_memory_mebibyte = self.__read_integer(galaxy_memory_mb_file) + properties[GALAXY_MEMORY_MB_KEY] = allocated_memory_mebibyte + + start_time_seconds = self.__read_seconds_since_epoch(job_directory, "start") + end_time_seconds = self.__read_seconds_since_epoch(job_directory, "end") + if start_time_seconds is not None and end_time_seconds is not None: + properties[START_EPOCH_KEY] = start_time_seconds + properties[END_EPOCH_KEY] = end_time_seconds + + runtime_seconds = end_time_seconds - start_time_seconds + properties[RUNTIME_SECONDS_KEY] = end_time_seconds - start_time_seconds + + if allocated_cpu_cores is not None: + tdp_per_ore = 115 / 10 + normalized_tdp_per_core = tdp_per_ore * allocated_cpu_cores + + memory_power_usage_constant = 0.3725 + memory_allocated_in_gibibyte = (allocated_memory_mebibyte or 0) / 1024 # Convert to gibibyte + + power_usage_effectiveness = 1.67 + + runtime_hours = runtime_seconds / (60 * 60) # Convert to hours + + power_needed_watts_cpu = power_usage_effectiveness * normalized_tdp_per_core + power_needed_watts_memory = ( + power_usage_effectiveness * memory_allocated_in_gibibyte * memory_power_usage_constant + ) + + energy_needed_cpu_kwh = (runtime_hours * power_needed_watts_cpu) / 1000 + energy_needed_memory_kwh = (runtime_hours * power_needed_watts_memory) / 1000 + + # Some jobs do not report memory usage, so a default value of 0 is used in this case + if energy_needed_memory_kwh != 0: + properties[ENERGY_NEEDED_MEMORY_KEY] = energy_needed_memory_kwh + + properties[ENERGY_NEEDED_CPU_KEY] = energy_needed_cpu_kwh + return properties def __record_galaxy_slots_command(self, job_directory): diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index 38258a175fa1..f19d2f3c5bbb 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -1021,25 +1021,13 @@ def materialize_to_history( return rval @router.get( - "/api/histories/{history_id}/metrics", - summary="Returns the metrics for all jobs for a given history.", + "/api/histories/{history_id}/energy_usage", + summary="Returns the energy usage of a given history.", ) - def get_metrics( + def get_energy_usage( self, history_id: HistoryIDPathParam, trans: ProvidesHistoryContext = DependsOnTrans, ): - """Get the cumulative metrics for all jobs in a history with ``history_id``.""" - return self.service.get_metrics(trans, history_id) - - @router.get( - "/api/histories/{history_id}/emissions", - summary="Returns the carbon emissions of a given history.", - ) - def get_emissions( - self, - history_id: HistoryIDPathParam, - trans: ProvidesHistoryContext = DependsOnTrans, - ): - """Get the carbon emissions data of a history with ``history_id``.""" - return self.service.get_emissions(trans, history_id) + """Get the energy usage data of a history with ``history_id``.""" + return self.service.get_energy_usage(trans, history_id) diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index 10e130a8781f..5481dc2bb079 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -1800,3 +1800,14 @@ def _deprecated_generate_bco( merge_history_metadata=merge_history_metadata or False, ) return self.invocations_service.deprecated_generate_invocation_bco(trans, invocation_id, export_options) + + @router.get( + "/api/invocations/{invocation_id}/energy_usage", + summary="Get the energy usage of a workflow invocation.", + ) + def get_energy_usage( + self, + invocation_id: InvocationIDPathParam, + trans: ProvidesUserContext = DependsOnTrans, + ): + return self.invocations_service.get_energy_usage(trans, invocation_id) diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index b996969cad18..7e1a1f1c154e 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -279,6 +279,7 @@ def app_pair(global_conf, load_app_kwds=None, wsgi_preflight=True, **kwargs): webapp.add_client_route("/interactivetool_entry_points/list") webapp.add_client_route("/libraries{path:.*?}") webapp.add_client_route("/storage{path:.*?}") + webapp.add_client_route("/statistics{path:.*?}") # ==== Done # Indicate that all configuration settings have been provided diff --git a/lib/galaxy/webapps/galaxy/services/history_contents.py b/lib/galaxy/webapps/galaxy/services/history_contents.py index b0a9752aa982..6a819785d8e3 100644 --- a/lib/galaxy/webapps/galaxy/services/history_contents.py +++ b/lib/galaxy/webapps/galaxy/services/history_contents.py @@ -1,7 +1,6 @@ import logging import os import re -import sqlalchemy as sa from typing import ( Any, cast, @@ -13,6 +12,7 @@ Union, ) +import sqlalchemy as sa from celery import chain from pydantic import ( ConfigDict, @@ -62,7 +62,6 @@ History, HistoryDatasetAssociation, HistoryDatasetCollectionAssociation, - Job, JobMetricNumeric, LibraryDataset, User, @@ -102,6 +101,7 @@ HistoryContentsResult, HistoryContentsWithStatsResult, HistoryContentType, + JobMetric, JobSourceType, MaterializeDatasetInstanceRequest, Model, @@ -859,118 +859,43 @@ def build_archive_files_and_paths(content, *parents): archive.write(file_path, archive_path) return archive - def get_metrics( - self, - trans, - history_id: DecodedDatabaseIdField, - ): - """Get the cumulative metrics for all jobs in a given history with ``history_id``. - - :param history_id: the encoded id of the history whose metrics are to be returned - """ - history = self._get_history(trans, history_id) - decoded_job_ids: List[int] = [job.id for job in history.jobs] - - total_runtime_in_seconds = trans.sa_session.query( - sa.func.sum(JobMetricNumeric.metric_value).label("total_runtime_in_seconds") - ).filter(JobMetricNumeric.job_id.in_(decoded_job_ids), JobMetricNumeric.metric_name == "runtime_seconds").scalar() - - total_memory_allocated_in_mebibyte = trans.sa_session.query( - sa.func.sum(JobMetricNumeric.metric_value).label("total_memory_allocated_in_mebibyte") - ).filter(JobMetricNumeric.job_id.in_(decoded_job_ids), JobMetricNumeric.metric_name == "galaxy_memory_mb").scalar() - - total_cores_allocated = trans.sa_session.query( - sa.func.sum(JobMetricNumeric.metric_value).label("total_cores_allocated") - ).filter(JobMetricNumeric.job_id.in_(decoded_job_ids), JobMetricNumeric.metric_name == "galaxy_slots").scalar() - - return { - "total_jobs_in_history": len(decoded_job_ids), - "total_runtime_in_seconds": total_runtime_in_seconds, - "total_cores_allocated": total_cores_allocated, - "total_memory_allocated_in_mebibyte": total_memory_allocated_in_mebibyte, - } - - def get_emissions( + def get_energy_usage( self, trans, history_id: DecodedDatabaseIdField, ): - """Get the carbon emissions of the history with ``history_id``. + """Get the energy usage of the history with ``history_id``. - :param history_id: the encoded id of the history whose carbon emissions should be returned + :param history_id: the encoded id of the history whose energy usage is to be calculated """ history = self._get_history(trans, history_id) decoded_job_ids: List[int] = [job.id for job in history.jobs] - history_carbon_emissions: dict[str, float] = { - "cpu_carbon_emissions": 0, - "memory_carbon_emissions": 0, - "total_carbon_emissions": 0, - "energy_needed_cpu": 0, - "energy_needed_memory": 0, - "total_energy_needed": 0, - } - - for job_id in decoded_job_ids: - job: Job = trans.sa_session.get(Job, job_id) - trans.sa_session.refresh(job) + total_energy_needed_cpu_kwh = ( + trans.sa_session.query(sa.func.sum(JobMetricNumeric.metric_value).label("energy_needed_cpu")) + .filter( + JobMetricNumeric.metric_name == "energy_needed_cpu", + JobMetricNumeric.job_id.in_(decoded_job_ids), + ) + .scalar() + ) - job_carbon_emissions = self.__get_job_carbon_emissions( - self.__get_job_metric_value("galaxy_memory_mb", job.metrics), - self.__get_job_metric_value("runtime_seconds", job.metrics), - self.__get_job_metric_value("galaxy_slots", job.metrics), + total_energy_needed_memory_kwh = ( + trans.sa_session.query(sa.func.sum(JobMetricNumeric.metric_value).label("energy_needed_memory")) + .filter( + JobMetricNumeric.metric_name == "energy_needed_memory", + JobMetricNumeric.job_id.in_(decoded_job_ids), ) + .scalar() + or 0.0 + ) - history_carbon_emissions["cpu_carbon_emissions"] += job_carbon_emissions["cpu_carbon_emissions"] - history_carbon_emissions["memory_carbon_emissions"] += job_carbon_emissions["memory_carbon_emissions"] - history_carbon_emissions["total_carbon_emissions"] += job_carbon_emissions["total_carbon_emissions"] - - history_carbon_emissions["energy_needed_cpu"] += job_carbon_emissions["energy_needed_cpu"] - history_carbon_emissions["energy_needed_memory"] += job_carbon_emissions["energy_needed_memory"] - history_carbon_emissions["total_energy_needed"] += job_carbon_emissions["total_energy_needed"] - - return history_carbon_emissions - - def __get_job_metric_value(self, key: str, metrics: List[JobMetricNumeric]) -> float: - for metric in metrics: - if metric.metric_name == key: - return float(metric.metric_value) - return 0 - - def __get_job_carbon_emissions( - self, memory_allocated_in_mebibyte: float, runtime_seconds: float, cores_allocated: float - ) -> dict[str, float]: - memory_power_usage = 0.3725 - runtime_in_hours = runtime_seconds / (60 * 60) # Convert to hours - memory_allocated_in_gibibyte = memory_allocated_in_mebibyte / 1024 # Convert to gibibyte - - tdp_per_ore = 115 / 10 - normalized_tdp_per_core = tdp_per_ore * cores_allocated - - # Power needed in Watt - power_usage_effectiveness = 1.67 - power_needed_cpu = power_usage_effectiveness * normalized_tdp_per_core - power_needed_memory = power_usage_effectiveness * memory_allocated_in_gibibyte * memory_power_usage - total_power_needed = power_needed_cpu + power_needed_memory - - # Energy needed. Convert Watt to kWh - energy_needed_cpu = (runtime_in_hours * power_needed_cpu) / 1000 - energy_needed_memory = (runtime_in_hours * power_needed_memory) / 1000 - total_energy_needed = (runtime_in_hours * total_power_needed) / 1000 - - # Carbon emissions (carbon intensity is in grams/kWh so emissions results are in grams of CO2) - carbon_intensity = 475.0 - cpu_carbon_emissions = energy_needed_cpu * carbon_intensity - memory_carbon_emissions = energy_needed_memory * carbon_intensity - total_carbon_emissions = total_energy_needed * carbon_intensity + total_energy_needed_kwh = float(total_energy_needed_cpu_kwh) + float(total_energy_needed_memory_kwh) return { - "cpu_carbon_emissions": cpu_carbon_emissions, - "memory_carbon_emissions": memory_carbon_emissions, - "total_carbon_emissions": total_carbon_emissions, - "energy_needed_cpu": energy_needed_cpu, - "energy_needed_memory": energy_needed_memory, - "total_energy_needed": total_energy_needed, + "total_energy_needed_cpu_kwh": total_energy_needed_cpu_kwh, + "total_energy_needed_memory_kwh": total_energy_needed_memory_kwh, + "total_energy_needed_kwh": total_energy_needed_kwh, } def __delete_dataset( diff --git a/lib/galaxy/webapps/galaxy/services/invocations.py b/lib/galaxy/webapps/galaxy/services/invocations.py index 79912285c2ac..fb52ed66259c 100644 --- a/lib/galaxy/webapps/galaxy/services/invocations.py +++ b/lib/galaxy/webapps/galaxy/services/invocations.py @@ -7,6 +7,7 @@ Tuple, ) +import sqlalchemy as sa from pydantic import Field from galaxy.celery.tasks import ( @@ -25,6 +26,7 @@ ) from galaxy.managers.workflows import WorkflowsManager from galaxy.model import ( + JobMetricNumeric, WorkflowInvocation, WorkflowInvocationStep, ) @@ -134,6 +136,41 @@ def show(self, trans, invocation_id, serialization_params, eager=False): wfi = self._workflows_manager.get_invocation(trans, invocation_id, eager) return self.serialize_workflow_invocation(wfi, serialization_params) + def get_energy_usage( + self, + trans, + invocation_id: DecodedDatabaseIdField, + ): + workflow_invocation = self._workflows_manager.get_invocation(trans, invocation_id, eager=True) + job_ids = [step.job_id for step in workflow_invocation.steps if step.job_id is not None] + + total_energy_needed_cpu_kwh = ( + trans.sa_session.query(sa.func.sum(JobMetricNumeric.metric_value).label("energy_needed_cpu")) + .filter( + JobMetricNumeric.metric_name == "energy_needed_cpu", + JobMetricNumeric.job_id.in_(job_ids), + ) + .scalar() + ) + + total_energy_needed_memory_kwh = ( + trans.sa_session.query(sa.func.sum(JobMetricNumeric.metric_value).label("energy_needed_memory")) + .filter( + JobMetricNumeric.metric_name == "energy_needed_memory", + JobMetricNumeric.job_id.in_(job_ids), + ) + .scalar() + or 0.0 + ) + + total_energy_needed_kwh = float(total_energy_needed_cpu_kwh) + float(total_energy_needed_memory_kwh) + + return { + "total_energy_needed_cpu_kwh": total_energy_needed_cpu_kwh, + "total_energy_needed_memory_kwh": total_energy_needed_memory_kwh, + "total_energy_needed_kwh": total_energy_needed_kwh, + } + def cancel(self, trans, invocation_id, serialization_params): wfi = self._workflows_manager.request_invocation_cancellation(trans, invocation_id) return self.serialize_workflow_invocation(wfi, serialization_params) From 6c78cf0a4db68d228e2046de4e498b591469f8ef Mon Sep 17 00:00:00 2001 From: Rendani Gangazhe Date: Tue, 20 Feb 2024 16:36:23 +0100 Subject: [PATCH 03/34] Format code --- .../History/Statistics/HistoryStatistics.vue | 3 +- .../WorkflowInvocationSummary.vue | 155 +++++++++--------- 2 files changed, 79 insertions(+), 79 deletions(-) diff --git a/client/src/components/History/Statistics/HistoryStatistics.vue b/client/src/components/History/Statistics/HistoryStatistics.vue index adc614f1048d..dc338c6d3dea 100644 --- a/client/src/components/History/Statistics/HistoryStatistics.vue +++ b/client/src/components/History/Statistics/HistoryStatistics.vue @@ -17,7 +17,8 @@ const { getHistoryNameById } = useHistoryStore();

Here is a summary of the statistics of the history - {{ getHistoryNameById(props.historyId) }}. + {{ getHistoryNameById(props.historyId) }}.

diff --git a/client/src/components/WorkflowInvocationState/WorkflowInvocationSummary.vue b/client/src/components/WorkflowInvocationState/WorkflowInvocationSummary.vue index dc5a5e320cd3..8e70e3726b88 100644 --- a/client/src/components/WorkflowInvocationState/WorkflowInvocationSummary.vue +++ b/client/src/components/WorkflowInvocationState/WorkflowInvocationSummary.vue @@ -18,13 +18,13 @@ import InvocationMessage from "@/components/WorkflowInvocationState/InvocationMe const getUrl = (path: any) => getRootFromIndexLink() + path; const { config } = useConfig(true); -const carbonIntensity = (config.value.carbon_intensity) ?? worldwideCarbonIntensity; -const geographicalServerLocationName = (config.value.geographical_server_location_name) ?? "GLOBAL"; +const carbonIntensity = config.value.carbon_intensity ?? worldwideCarbonIntensity; +const geographicalServerLocationName = config.value.geographical_server_location_name ?? "GLOBAL"; const energyUsage = ref({ - total_energy_needed_cpu_kwh: 0, - total_energy_needed_memory_kwh: 0, - total_energy_needed_kwh: 0, + total_energy_needed_cpu_kwh: 0, + total_energy_needed_memory_kwh: 0, + total_energy_needed_kwh: 0, }); const props = defineProps({ @@ -50,10 +50,9 @@ const props = defineProps({ required: false, default: null, }, -}) -const stepStatesInterval = null; -const jobStatesInterval = null; -const reportTooltip = "View report for this workflow invocation"; +}); + +const reportTooltip = "View report for this workflow invocation"; const generatePdfTooltip = "Generate PDF report for this workflow invocation"; const invocationId = computed(() => { @@ -69,18 +68,18 @@ const indexStr = computed(() => { const invocationState = computed(() => { return props.invocation?.state || "new"; -}) +}); const invocationStateSuccess = computed(() => { return invocationState.value === "scheduled" && runningCount.value === 0 && props.invocationAndJobTerminal; -}) +}); const disabledReportTooltip = computed(() => { const state = invocationState.value; const runCount = runningCount.value; if (invocationState.value != "scheduled") { - return `This workflow is not currently scheduled. The current state is ${state}. Once the workflow is fully scheduled and jobs have complete this option will become available.` + return `This workflow is not currently scheduled. The current state is ${state}. Once the workflow is fully scheduled and jobs have complete this option will become available.`; } if (runCount != 0) { @@ -112,15 +111,15 @@ const stepStates = computed(() => { const invocationLink = computed(() => { return getUrl(`workflows/invocations/report?id=${invocationId.value}`); -}) +}); const invocationPdfLink = computed(() => { return getUrl(`api/invocations/${invocationId.value}/report.pdf`); -}) +}); const stepStatesStr = computed(() => { return `${stepStates.value?.scheduled || 0} of ${stepCount.value} steps successfully scheduled.`; -}) +}); const jobCount = computed(() => { return !props.jobStatesSummary ? null : props.jobStatesSummary.jobCount(); @@ -132,25 +131,25 @@ const jobStatesStr = computed(() => { jobStr += " (total number of jobs will change until all steps fully scheduled)"; } return `${jobStr}.`; -}) +}); const runningCount = computed(() => { return countStates(["running"]); -}) +}); const okCount = computed(() => { return countStates(["ok", "skipped"]); -}) +}); const errorCount = computed(() => { return countStates(["error", "deleted"]); -}) +}); const newCount = computed(() => { return jobCount.value - okCount.value - runningCount.value - errorCount.value; -}) +}); -const emit = defineEmits(['invocation-cancelled']); +const emit = defineEmits(["invocation-cancelled"]); function onCancel() { - emit('invocation-cancelled'); + emit("invocation-cancelled"); } function countStates(states: string[]) { @@ -164,13 +163,13 @@ function countStates(states: string[]) { } async function fetchEnergyUsageData() { - const res = await fetcher.path("/api/invocations/{invocation_id}/energy_usage").method("get").create()({ - invocation_id: props.invocation?.id - }); + const res = await fetcher.path("/api/invocations/{invocation_id}/energy_usage").method("get").create()({ + invocation_id: props.invocation?.id, + }); - if (res.ok) { - energyUsage.value = res.data; - } + if (res.ok) { + energyUsage.value = res.data; + } } onMounted(() => { @@ -261,56 +260,56 @@ watch( class="jobs-progress" />

- Carbon Emissions: - - - - - - + Carbon Emissions: + + + + + +
From 59902c025cc7ed7ff32c1d7fc200b62626a28e33 Mon Sep 17 00:00:00 2001 From: Rendani Gangazhe Date: Tue, 20 Feb 2024 17:04:04 +0100 Subject: [PATCH 04/34] Unify energy usage summary type in API schema --- client/src/api/index.ts | 9 ++--- client/src/api/schema/schema.ts | 35 +++++-------------- .../Statistics/HistoryCarbonEmissions.vue | 4 +-- .../WorkflowInvocationSummary.vue | 4 +-- .../webapps/galaxy/services/invocations.py | 6 ++-- 5 files changed, 17 insertions(+), 41 deletions(-) diff --git a/client/src/api/index.ts b/client/src/api/index.ts index e2d96d3b5c37..2dee873535d7 100644 --- a/client/src/api/index.ts +++ b/client/src/api/index.ts @@ -18,14 +18,9 @@ export type HistoryDetailed = components["schemas"]["HistoryDetailed"]; export type HistoryContentItemBase = components["schemas"]["EncodedHistoryContentItem"]; /** - * Contains information about the energy usage of a History. + * Contains information about the energy usage of a history or an workflow invocation. */ -export type HistoryEnergyUsage = components["schemas"]["HistoryEnergyUsage"]; - -/** - * Contains information about the energy usage of a given workflow invocation. - */ -export type WorkflowInvocationEnergyUsage = components["schemas"]["WorkflowInvocationEnergyUsage"]; +export type EnergyUsageSummary = components["schemas"]["EnergyUsageSummary"]; /** * Contains summary information about a HistoryDatasetAssociation. diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 43ecdefbc93e..63a9798cdca7 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -6543,25 +6543,25 @@ export interface components { [key: string]: unknown | undefined; }; /** - * History Metrics - * @description The accumulated metrics of all jobs in a history. + * Energy Usage Summary Metrics + * @description The summary of energy usage for all jobs in a given history or a workflow invocation. */ - HistoryEnergyUsage: { + EnergyUsageSummary: { /** * Total Energy Needed CPU (kWh) - * @description The total energy used by the CPU for all jobs in the history in kilowatt hours. + * @description The total energy used by the CPU in kilowatt hours. */ total_energy_needed_cpu_kwh: number; /** * Total Energy Needed Memory (kWh) - * @description The total energy used by memory for all jobs in the history in kilowatt hours. + * @description The total energy used by memory in kilowatt hours. */ total_energy_needed_memory_kwh: number; /** * Total Energy Needed (kWh) - * @description The total energy needed to run all jobs in the history in kilowatt hours. + * @description The total energy needed to run all jobs in kilowatt hours. */ total_energy_needed_kwh: number; }; @@ -7370,25 +7370,6 @@ export interface components { */ action: boolean; }; - WorkflowInvocationEnergyUsage: { - /** - * Total Energy Needed CPU (kWh) - * @description The total energy used by the CPU for all jobs in the workflow invocation in kilowatt hours. - */ - total_energy_needed_cpu_kwh: number; - - /** - * Total Energy Needed Memory (kWh) - * @description The total energy used by memory for all jobs in the workflow invocation in kilowatt hours. - */ - total_energy_needed_memory_kwh: number; - - /** - * Total Energy Needed (kWh) - * @description The total energy needed to run all jobs in the workflow invocation in kilowatt hours. - */ - total_energy_needed_kwh: number; - }; /** * ItemTagsCreatePayload * @description Payload schema for creating an item tag. @@ -14472,7 +14453,7 @@ export interface operations { /** @description Successful Response */ 200: { content: { - "application/json": components["schemas"]["HistoryEnergyUsage"]; + "application/json": components["schemas"]["EnergyUsageSummary"]; }; }; /** @description Validation Error */ @@ -17116,7 +17097,7 @@ export interface operations { /** @description Successful Response */ 200: { content: { - "application/json": components["schemas"]["WorkflowInvocationEnergyUsage"]; + "application/json": components["schemas"]["EnergyUsageSummary"]; }; }; /** @description Validation Error */ diff --git a/client/src/components/History/Statistics/HistoryCarbonEmissions.vue b/client/src/components/History/Statistics/HistoryCarbonEmissions.vue index d2fa4e2e1706..be9f5e82d988 100644 --- a/client/src/components/History/Statistics/HistoryCarbonEmissions.vue +++ b/client/src/components/History/Statistics/HistoryCarbonEmissions.vue @@ -4,7 +4,7 @@ import { faQuestionCircle } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome"; import { onMounted, ref, watch } from "vue"; -import { HistoryEnergyUsage } from "@/api"; +import { EnergyUsageSummary } from "@/api"; import { fetcher } from "@/api/schema"; import { worldwideCarbonIntensity, @@ -23,7 +23,7 @@ const { config } = useConfig(true); const carbonIntensity = (config.value.carbon_intensity as number) ?? worldwideCarbonIntensity; const geographicalServerLocationName = (config.value.geographical_server_location_name as string) ?? "GLOBAL"; -const energyUsage = ref({ +const energyUsage = ref({ total_energy_needed_cpu_kwh: 0, total_energy_needed_memory_kwh: 0, total_energy_needed_kwh: 0, diff --git a/client/src/components/WorkflowInvocationState/WorkflowInvocationSummary.vue b/client/src/components/WorkflowInvocationState/WorkflowInvocationSummary.vue index 8e70e3726b88..697ed7290714 100644 --- a/client/src/components/WorkflowInvocationState/WorkflowInvocationSummary.vue +++ b/client/src/components/WorkflowInvocationState/WorkflowInvocationSummary.vue @@ -1,7 +1,7 @@ diff --git a/client/src/components/JobMetrics/JobMetrics.vue b/client/src/components/JobMetrics/JobMetrics.vue index f8ad722156f0..beebc4b7f156 100644 --- a/client/src/components/JobMetrics/JobMetrics.vue +++ b/client/src/components/JobMetrics/JobMetrics.vue @@ -1,10 +1,7 @@ diff --git a/client/src/components/Markdown/Elements/JobMetrics.vue b/client/src/components/Markdown/Elements/JobMetrics.vue index 95571896863a..b17ca9529f60 100644 --- a/client/src/components/Markdown/Elements/JobMetrics.vue +++ b/client/src/components/Markdown/Elements/JobMetrics.vue @@ -69,13 +69,7 @@ watch( :job-id="jobId" :implicit-collection-jobs-id="implicitCollectionJobsId" :select-job-options="selectJobOptions"> - + {{ footer }} diff --git a/client/src/components/WorkflowInvocationState/WorkflowInvocationSummary.vue b/client/src/components/WorkflowInvocationState/WorkflowInvocationSummary.vue index 697ed7290714..ec6e0365a39a 100644 --- a/client/src/components/WorkflowInvocationState/WorkflowInvocationSummary.vue +++ b/client/src/components/WorkflowInvocationState/WorkflowInvocationSummary.vue @@ -301,12 +301,6 @@ watch( 2. Using the global default power usage effectiveness value of {{ worldwidePowerUsageEffectiveness }}. - -
- - 3. based off of the closest AWS EC2 instance comparable to the server that ran - this job. Estimates depend on the core count, allocated memory and the job runtime. The closest - estimate is a TODO instance.

diff --git a/lib/galaxy/job_metrics/instrumenters/core.py b/lib/galaxy/job_metrics/instrumenters/core.py index 4abce962197c..d17335da4daa 100644 --- a/lib/galaxy/job_metrics/instrumenters/core.py +++ b/lib/galaxy/job_metrics/instrumenters/core.py @@ -34,6 +34,8 @@ ENERGY_NEEDED_MEMORY_KEY = "energy_needed_memory" ENERGY_NEEDED_CPU_KEY = "energy_needed_cpu" +ESTIMATED_SERVER_INSTANCE_NAME_KEY = "estimated_server_instance_name" + class CorePluginFormatter(JobMetricFormatter): def format(self, key: str, value: Any) -> FormattedMetric: @@ -47,6 +49,8 @@ def format(self, key: str, value: Any) -> FormattedMetric: return FormattedMetric("CPU Energy Usage", self.__format_energy_needed_text(float(value))) elif key == ENERGY_NEEDED_MEMORY_KEY: return FormattedMetric("Memory Energy Usage", self.__format_energy_needed_text(float(value))) + elif key == ESTIMATED_SERVER_INSTANCE_NAME_KEY: + return FormattedMetric("Estimated Server Instance", str(value)) else: # TODO: Use localized version of this from galaxy.ini title = "Job Start Time" if key == START_EPOCH_KEY else "Job End Time" @@ -157,6 +161,7 @@ def job_properties(self, job_id, job_directory: str) -> Dict[str, Any]: properties[ENERGY_NEEDED_MEMORY_KEY] = energy_needed_memory_kwh properties[ENERGY_NEEDED_CPU_KEY] = energy_needed_cpu_kwh + properties[ESTIMATED_SERVER_INSTANCE_NAME_KEY] = estimated_server_instance["name"] return properties @@ -197,12 +202,12 @@ def __get_estimated_server_instance( server_instance = None for aws_instance in load_aws_ec2_reference_data_json(): - # Exclude memory from search criteria + # Use only core count in search criteria if adjusted_memory == 0 and aws_instance["v_cpu_count"] >= allocated_cpu_cores: server_instance = aws_instance break - # Search by all criteria + # Use both core count and allocated memory in search criteria if aws_instance["mem"] >= adjusted_memory and aws_instance["v_cpu_count"] >= allocated_cpu_cores: server_instance = aws_instance break From 9e64c04201daae7a1001e025465b59cab5de7a41 Mon Sep 17 00:00:00 2001 From: Rendani Gangazhe Date: Wed, 21 Feb 2024 12:39:37 +0100 Subject: [PATCH 07/34] Do not display energy usage metrics in job metrics list --- client/src/components/JobMetrics/JobMetrics.vue | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/client/src/components/JobMetrics/JobMetrics.vue b/client/src/components/JobMetrics/JobMetrics.vue index beebc4b7f156..f4ae42394ebe 100644 --- a/client/src/components/JobMetrics/JobMetrics.vue +++ b/client/src/components/JobMetrics/JobMetrics.vue @@ -45,8 +45,17 @@ const jobMetrics = computed(() => { const jobMetricsGroupedByPluginType = computed(() => { const pluginGroups: Record = {}; + const ignoredMetrics = [ + "energy_needed_cpu", + "energy_needed_memory", + "estimated_server_instance_name", + ]; for (const metric of jobMetrics.value) { + if (ignoredMetrics.includes(metric.name)) { + continue; + } + // new group found if (!(metric.plugin in pluginGroups)) { pluginGroups[metric.plugin] = {}; From bcce348a376b8abc88abd7f26b6900b22880386d Mon Sep 17 00:00:00 2001 From: Rendani Gangazhe Date: Wed, 21 Feb 2024 13:05:41 +0100 Subject: [PATCH 08/34] Clean up carbon emissions client markup and code --- .../CarbonEmissions/CarbonEmissions.vue | 8 ++++- .../Statistics/HistoryCarbonEmissions.vue | 1 - .../JobMetrics/JobCarbonEmissions.vue | 29 +++++++++---------- .../src/components/JobMetrics/JobMetrics.vue | 8 ++--- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/client/src/components/CarbonEmissions/CarbonEmissions.vue b/client/src/components/CarbonEmissions/CarbonEmissions.vue index 2d5a9e8decd9..4c9bbd6d4316 100644 --- a/client/src/components/CarbonEmissions/CarbonEmissions.vue +++ b/client/src/components/CarbonEmissions/CarbonEmissions.vue @@ -300,7 +300,9 @@ function getEnergyNeededText(energyNeededInKiloWattHours: number) { - +
+ +
@@ -325,4 +327,8 @@ function getEnergyNeededText(energyNeededInKiloWattHours: number) { grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); grid-gap: 1rem; } + +.footer { + align-self: flex-start; +} diff --git a/client/src/components/History/Statistics/HistoryCarbonEmissions.vue b/client/src/components/History/Statistics/HistoryCarbonEmissions.vue index 6c4f4e2abca4..6d288e178318 100644 --- a/client/src/components/History/Statistics/HistoryCarbonEmissions.vue +++ b/client/src/components/History/Statistics/HistoryCarbonEmissions.vue @@ -19,7 +19,6 @@ library.add(faQuestionCircle); const props = defineProps<{ historyId: string }>(); const { config } = useConfig(true); - const carbonIntensity = (config.value.carbon_intensity as number) ?? worldwideCarbonIntensity; const geographicalServerLocationName = (config.value.geographical_server_location_name as string) ?? "GLOBAL"; diff --git a/client/src/components/JobMetrics/JobCarbonEmissions.vue b/client/src/components/JobMetrics/JobCarbonEmissions.vue index 1bfe21108834..5e961bd24710 100644 --- a/client/src/components/JobMetrics/JobCarbonEmissions.vue +++ b/client/src/components/JobMetrics/JobCarbonEmissions.vue @@ -13,21 +13,19 @@ import Heading from "@/components/Common/Heading.vue"; library.add(faQuestionCircle); -interface JobCarbonEmissionsProps { +const props = defineProps<{ energyUsage: { energyNeededCPU: number; energyNeededMemory: number; }; estimatedServerInstanceName: string; -} - -const props = defineProps(); - -const totalEnergyNeeded = ref(props.energyUsage.energyNeededCPU + props.energyUsage.energyNeededMemory); +}>(); const { config } = useConfig(true); const carbonIntensity = (config.value.carbon_intensity as number) ?? worldwideCarbonIntensity; const geographicalServerLocationName = (config.value.geographical_server_location_name as string) ?? "GLOBAL"; + +const totalEnergyNeeded = ref(props.energyUsage.energyNeededCPU + props.energyUsage.energyNeededMemory); diff --git a/client/src/components/JobMetrics/JobMetrics.vue b/client/src/components/JobMetrics/JobMetrics.vue index f4ae42394ebe..0f3b5e642bfa 100644 --- a/client/src/components/JobMetrics/JobMetrics.vue +++ b/client/src/components/JobMetrics/JobMetrics.vue @@ -45,15 +45,11 @@ const jobMetrics = computed(() => { const jobMetricsGroupedByPluginType = computed(() => { const pluginGroups: Record = {}; - const ignoredMetrics = [ - "energy_needed_cpu", - "energy_needed_memory", - "estimated_server_instance_name", - ]; + const ignoredMetrics = ["energy_needed_cpu", "energy_needed_memory", "estimated_server_instance_name"]; for (const metric of jobMetrics.value) { if (ignoredMetrics.includes(metric.name)) { - continue; + continue; } // new group found From 4bdee2aa0c4cf34110a176a5178443cc2cc9ac75 Mon Sep 17 00:00:00 2001 From: Rendani Gangazhe Date: Wed, 21 Feb 2024 15:07:15 +0100 Subject: [PATCH 09/34] Revert HistoryView component to previous state using options API --- client/src/components/History/HistoryView.vue | 221 ++++++++---------- 1 file changed, 93 insertions(+), 128 deletions(-) diff --git a/client/src/components/History/HistoryView.vue b/client/src/components/History/HistoryView.vue index 3b089308b108..f03c02e718d8 100644 --- a/client/src/components/History/HistoryView.vue +++ b/client/src/components/History/HistoryView.vue @@ -1,129 +1,8 @@ - - + + From dcfdbc81d54059d6d1bdf2e53dde00ffbda7ea9a Mon Sep 17 00:00:00 2001 From: Rendani Gangazhe Date: Wed, 21 Feb 2024 20:01:04 +0100 Subject: [PATCH 10/34] Use correct props and adjust code style --- .../components/CarbonEmissions/CarbonEmissions.vue | 11 ++++------- client/src/components/History/HistoryView.vue | 4 ++-- .../components/History/Multiple/MultipleViewItem.vue | 2 +- .../History/Statistics/HistoryCarbonEmissions.vue | 5 +---- .../src/components/JobMetrics/JobCarbonEmissions.vue | 9 ++------- .../WorkflowInvocationSummary.vue | 4 +--- 6 files changed, 11 insertions(+), 24 deletions(-) diff --git a/client/src/components/CarbonEmissions/CarbonEmissions.vue b/client/src/components/CarbonEmissions/CarbonEmissions.vue index 4c9bbd6d4316..ca591669d142 100644 --- a/client/src/components/CarbonEmissions/CarbonEmissions.vue +++ b/client/src/components/CarbonEmissions/CarbonEmissions.vue @@ -2,22 +2,19 @@ import { GetComponentPropTypes } from "types/utilityTypes"; import { computed, unref } from "vue"; -import { worldwideCarbonIntensity } from "@/components/CarbonEmissions/carbonEmissionConstants"; -import * as carbonEmissionsConstants from "@/components/CarbonEmissions/carbonEmissionConstants.js"; import { useConfig } from "@/composables/config"; -import CarbonEmissionsCard from "@/components/CarbonEmissions/CarbonEmissionCard.vue"; +import * as carbonEmissionsConstants from "./carbonEmissionConstants.js"; + +import CarbonEmissionsCard from "./CarbonEmissionCard.vue"; const props = defineProps<{ energyNeededCPU: number; energyNeededMemory: number; - totalEnergyNeeded: number; - totalCarbonEmissions: number; }>(); const { config } = useConfig(true); - -const carbonIntensity = (config.value.carbon_intensity as number) ?? worldwideCarbonIntensity; +const carbonIntensity = (config.value.carbon_intensity as number) ?? carbonEmissionsConstants.worldwideCarbonIntensity; const canShowMemory = computed(() => { return props.energyNeededMemory && props.energyNeededMemory !== 0; diff --git a/client/src/components/History/HistoryView.vue b/client/src/components/History/HistoryView.vue index f03c02e718d8..7f90623be5ed 100644 --- a/client/src/components/History/HistoryView.vue +++ b/client/src/components/History/HistoryView.vue @@ -33,8 +33,8 @@ diff --git a/client/src/components/History/Multiple/MultipleViewItem.vue b/client/src/components/History/Multiple/MultipleViewItem.vue index e69beea40780..44b5d635cb49 100644 --- a/client/src/components/History/Multiple/MultipleViewItem.vue +++ b/client/src/components/History/Multiple/MultipleViewItem.vue @@ -15,7 +15,7 @@ v-else :history="getHistory" :filter="filter" - :show-controls="false" + :should-show-controls="false" @view-collection="onViewCollection" />
diff --git a/client/src/components/History/Statistics/HistoryCarbonEmissions.vue b/client/src/components/History/Statistics/HistoryCarbonEmissions.vue index 6d288e178318..cb67da345fd8 100644 --- a/client/src/components/History/Statistics/HistoryCarbonEmissions.vue +++ b/client/src/components/History/Statistics/HistoryCarbonEmissions.vue @@ -19,7 +19,6 @@ library.add(faQuestionCircle); const props = defineProps<{ historyId: string }>(); const { config } = useConfig(true); -const carbonIntensity = (config.value.carbon_intensity as number) ?? worldwideCarbonIntensity; const geographicalServerLocationName = (config.value.geographical_server_location_name as string) ?? "GLOBAL"; const energyUsage = ref({ @@ -57,9 +56,7 @@ watch(
+ :energy-needed-c-p-u="energyUsage.total_energy_needed_cpu_kwh">