Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Updates to setup script #274

Draft
wants to merge 4 commits into
base: main
Choose a base branch
from
Draft

Updates to setup script #274

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9cc7924
Remove Prometheus-specific configuration for PCP setup
connersaeli Sep 13, 2023
d8c43d3
Use pymongo instead of mongo shell to populate schema collection
Mar 8, 2024
fb2038e
Merge remote-tracking branch 'fork/prom-resconf-fix' into mongosh-bugfix
Mar 8, 2024
667f533
Use blank username and password for defaults
Mar 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .circleci/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ jobs:
install-type:
type: string
docker:
- image: tools-ext-01.ccr.xdmod.org/xdmod-job_performance-10.0.0:rockylinux8-0.1
- image: tools-ext-01.ccr.xdmod.org/xdmod-job_performance-10.5.0:rockylinux8-0.1
environment:
TERM: xterm
TERMINFO: /bin/bash
Expand All @@ -32,7 +32,7 @@ jobs:
XDMOD_IS_CORE: yes
XDMOD_INSTALL_DIR: /xdmod
XDMOD_TEST_MODE: << parameters.test-mode >>
SUPREMM_INSTALL_TYPE: << parameters.install-type >>
SUPREMM_INSTALL_TYPE: << parameters.install-type >>
steps:
- checkout
- run:
Expand Down
2 changes: 1 addition & 1 deletion setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
package_dir={'': 'src'},
packages=find_packages(where='src'),
package_data={
'supremm': ['assets/modw_supremm.sql', 'assets/mongo_setup.js', '*.pxd', '*.pyx'],
'supremm': ['assets/modw_supremm.sql', 'assets/*schema.json', '*.pxd', '*.pyx'],
'supremm.datasource.pcp.pcpcinterface': ['*.pxd', '*.pyx']
},
data_files=[
Expand Down
357 changes: 357 additions & 0 deletions src/supremm/assets/schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,357 @@
{
"summary_version": "summary-1.0.6",
"_id": "summary-1.0.6",
"definitions": {
"lnet": {
"documentation": "",
"type": "",
"unit": ""
},
"catastrophe": {
"documentation": "",
"type": "",
"unit": ""
},
"infiniband": {
"documentation": "",
"type": "",
"unit": ""
},
"cpuperf": {
"cpiref": {
"documentation": "The average clock ticks per instruction for each core.",
"type": "ratio",
"unit": "ratio"
},
"cpldref": {
"documentation": "The average clock ticks per L1D cache load for each core.",
"type": "ratio",
"unit": "ratio"
},
"flops": {
"documentation": "The number of floating point instructions executed per core.",
"type": "instant",
"unit": "op"
}
},
"load1": {
"max": {
"documentation": "The maximum value of the 1-second 'load average' during the job.",
"type": "instant",
"unit": ""
},
"maxpercore": {
"documentation": "The maximum value of the 1-second 'load average' per core during the job.",
"type": "instant",
"unit": ""
},
"mean": {
"documentation": "The arithmetic mean of the 1-second 'load average' during the job.",
"type": "instant",
"unit": ""
},
"meanpercore": {
"documentation": "The arithmetic mean of the 1-second 'load average' per core during the job.",
"type": "instant",
"unit": ""
}
},
"lustre": {
"*": {
"read_bytes-total": {
"documentation": "",
"type": "instant",
"unit": "byte"
},
"write_bytes-total": {
"documentation": "",
"type": "instant",
"unit": "byte"
}
}
},
"gpfs": {
"*": {
"read": {
"documentation": "The number of read operations.",
"type": "instant",
"unit": ""
},
"write": {
"documentation": "The number of write operations",
"type": "instant",
"unit": ""
},
"read_bytes": {
"documentation": "The amount of data read.",
"type": "instant",
"unit": "byte"
},
"write_bytes": {
"documentation": "The amount of data written.",
"type": "instant",
"unit": "byte"
}
}
},
"nodememory": {
"free": {
"documentation": "The average amount of free memory per node for the job. The value is obtained from /proc/meminfo. The average is calculated as the mean value of each memory usage measurement.",
"type": "instant",
"unit": "kilobyte"
},
"maxfree": {
"documentation": "The maximum value of the free memory on a node.",
"type": "instant",
"unit": "kilobyte"
},
"used": {
"documentation": "The average amount of used memory per node.",
"type": "instant",
"unit": "kilobyte"
},
"maxused": {
"documentation": "The maximum value of the used memory on a node.",
"type": "instant",
"unit": "kilobyte"
},
"physmem": {
"documentation": "The total physical memory on a node.",
"type": "instant",
"unit": "kilobyte"
}
},
"process_memory": {
"usage": {
"avg": {
"documentation": "The average amount of memory used in the memory cgroup that contained the job. The value is obtained from the kernel cgroup metrics. The average is calculated as the mean value of each memory usage measurement.",
"type": "instant",
"unit": "byte"
},
"max": {
"documentation": "The maximum value of the process memory on a node.",
"type": "instant",
"unit": "byte"
}
},
"usageratio": {
"avg": {
"documentation": "The average ratio of memory used to the memory limit for the processes in the memory cgroup that contained the job. The value is obtained from the kernel cgroup metrics.",
"type": "instant",
"unit": "ratio"
},
"max": {
"documentation": "The maximum value of the ratio of memory used to the memory limit for the processes in the memory cgroup that contained the job. The value is obtained from the kernel cgroup metrics.",
"type": "instant",
"unit": "ratio"
}
},
"limit": {
"documentation": "The memory limit for the memory cgroup that contained the job. The value is obtained from the kernel cgroup metrics.",
"type": "instant",
"unit": "byte"
}
},
"nfs": {
"documentation": "",
"type": "",
"unit": ""
},
"simdins": {
"documentation": "",
"type": "",
"unit": ""
},
"uncperf": {
"membw": {
"documentation": "The average amount of data transferred to and from main memory per node.",
"type": "instant",
"unit": "byte"
}
},
"memory": {
"used": {
"documentation": "The mean memory usage during the job. This value is the usage per core. This value includes the OS buffers and kernel slab cache.",
"type": "",
"unit": "kilobyte"
},
"used_minus_cache": {
"documentation": "The mean memory usage during the job. This value is the usage per core.",
"type": "",
"unit": "kilobyte"
}
},
"gpu": {
"*": {
"gpuactive": {
"documentation": "The average GPU usage.",
"type": "instant",
"unit": "%"
},
"gpuactivemax": {
"documentation": "The peak GPU usage.",
"type": "instant",
"unit": "%"
},
"memused": {
"documentation": "The average memory usage per GPU.",
"type": "instant",
"unit": "byte"
},
"memusedmax": {
"documentation": "The peak memory usage for each GPU.",
"type": "instant",
"unit": "byte"
}
}
},
"proc": {
"documentation": "",
"type": "",
"unit": ""
},
"cpu": {
"jobcpus": {
"idle": {
"documentation": "The CPU usage in idle mode of the cores that were assigned to the job. This metric reports the overall usage of each core that the job was assigned rather than, for example, the CPU usage of the job processes themselves.",
"type": "instant",
"unit": "ratio"
},
"iowait": {
"documentation": "The CPU usage in iowait mode of the cores that were assigned to the job. This metric reports the overall usage of each core that the job was assigned rather than, for example, the CPU usage of the job processes themselves.",
"type": "instant",
"unit": "ratio"
},
"irq": {
"documentation": "The CPU usage in irq mode of the cores that were assigned to the job. This metric reports the overall usage of each core that the job was assigned rather than, for example, the CPU usage of the job processes themselves.",
"type": "instant",
"unit": "ratio"
},
"nice": {
"documentation": "The CPU usage in nice mode of the cores that were assigned to the job. This metric reports the overall usage of each core that the job was assigned rather than, for example, the CPU usage of the job processes themselves.",
"type": "instant",
"unit": "ratio"
},
"softirq": {
"documentation": "The CPU usage in softirq mode of the cores that were assigned to the job. This metric reports the overall usage of each core that the job was assigned rather than, for example, the CPU usage of the job processes themselves.",
"type": "instant",
"unit": "ratio"
},
"system": {
"documentation": "The CPU usage in system mode of the cores that were assigned to the job. This metric reports the overall usage of each core that the job was assigned rather than, for example, the CPU usage of the job processes themselves.",
"type": "instant",
"unit": "ratio"
},
"user": {
"documentation": "The CPU usage in user mode of the cores that were assigned to the job. This metric reports the overall usage of each core that the job was assigned rather than, for example, the CPU usage of the job processes themselves.",
"type": "instant",
"unit": "ratio"
}
},
"nodecpus": {
"idle": {
"documentation": "The CPU usage in idle mode of the compute nodes on which the job ran. This value includes the contribution from all cores on each compute node regardless of whether the job processes were assigned to or ran on them.",
"type": "instant",
"unit": "ratio"
},
"iowait": {
"documentation": "The CPU usage in iowait mode of the compute nodes on which the job ran. This value includes the contribution from all cores on each compute node regardless of whether the job processes were assigned to or ran on them.",
"type": "instant",
"unit": "ratio"
},
"irq": {
"documentation": "The CPU usage in irq mode of the compute nodes on which the job ran. This value includes the contribution from all cores on each compute node regardless of whether the job processes were assigned to or ran on them.",
"type": "instant",
"unit": "ratio"
},
"nice": {
"documentation": "The CPU usage in nice mode of the compute nodes on which the job ran. This value includes the contribution from all cores on each compute node regardless of whether the job processes were assigned to or ran on them.",
"type": "instant",
"unit": "ratio"
},
"softirq": {
"documentation": "The CPU usage in softirq mode of the compute nodes on which the job ran. This value includes the contribution from all cores on each compute node regardless of whether the job processes were assigned to or ran on them.",
"type": "instant",
"unit": "ratio"
},
"system": {
"documentation": "The CPU usage in system mode of the compute nodes on which the job ran. This value includes the contribution from all cores on each compute node regardless of whether the job processes were assigned to or ran on them.",
"type": "instant",
"unit": "ratio"
},
"user": {
"documentation": "The CPU usage in user mode of the compute nodes on which the job ran. This value includes the contribution from all cores on each compute node regardless of whether the job processes were assigned to or ran on them.",
"type": "instant",
"unit": "ratio"
}
},
"effectivecpus": {
"user": {
"documentation": "The effective cpu metric reports the statistics of the subset of CPU cores that have an average usage above a threshold. The metric is intended to be used to distinguish cpu cores that are running user processes from those that are not. The threshold value is resource-specific.",
"type": "instant",
"unit": "ratio"
}
}
},
"block": {
"*": {
"read": {
"documentation": "The number of disk read operations.",
"type": "instant",
"unit": ""
},
"write": {
"documentation": "The number of disk write operations",
"type": "instant",
"unit": ""
},
"read_bytes": {
"documentation": "The amount of data read from disk.",
"type": "instant",
"unit": "byte"
},
"write_bytes": {
"documentation": "The amount of data written to disk.",
"type": "instant",
"unit": "byte"
}
}
},
"gemini": {
"totalinput": {
"documentation": "",
"type": "instant",
"unit": "byte"
},
"totaloutput_optA": {
"documentation": "",
"type": "instant",
"unit": "byte"
},
"totaloutput_optB": {
"documentation": "",
"type": "instant",
"unit": "byte"
}
},
"network": {
"*": {
"in-bytes": {
"documentation": "",
"type": "instant",
"unit": "byte"
},
"out-bytes": {
"documentation": "",
"type": "instant",
"unit": "byte"
}
}
},
"cpucategories": {
"documentation": "Categorization of the CPU utilization of the job as good, pinned, unpinned, or low. A job is good if every core is heavily utilized, and a job is low if the cores are barely utilized or not at all. A pinned job consists of a scenario where a subset of the cores does most of the work, while an unpinned job is when the work is spread unevenly across many cores.",
"type": "",
"unit": ""
}
}
}
Loading