Skip to content

updated dump to filter on pid #1354

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Spurs20 merged 2 commits into from
Jan 21, 2026
Merged

updated dump to filter on pid #1354

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b6ddf9f
updated dump to filter on pid
Mletter1 Jan 10, 2026
ce589cf
Update docker-compose.yml
Mletter1 Jan 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docker/compose/slycat-compose/docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ services:
COUCHDB_PASSWORD: password
networks:
- sly-net
#uncomment to expose cherrypy @http://localhost:5984/_utils
# uncomment to expose cherrypy @http://localhost:5984/_utils
# ports:
# - "5984:5984"
volumes:
Expand Down
170 changes: 140 additions & 30 deletions web-server/slycat-dump.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,24 +15,49 @@
parser = argparse.ArgumentParser()
parser.add_argument("output_dir", help="Directory where results will be stored.")
parser.add_argument("--all-projects", action="store_true", help="Dump all projects.")
parser.add_argument("--all-references-bookmarks", action="store_true", help="Dump all projects.")
parser.add_argument("--couchdb-database", default="slycat", help="CouchDB database. Default: %(default)s")
parser.add_argument("--couchdb-host", default="localhost", help="CouchDB host. Default: %(default)s")
parser.add_argument("--couchdb-port", type=int, default=5984, help="CouchDB port. Default: %(default)s")
parser.add_argument("--data-store", default="data-store", help="Path to the hdf5 data storage directory. Default: %(default)s")
parser.add_argument(
"--all-references-bookmarks", action="store_true", help="Dump all projects."
)
parser.add_argument(
"--couchdb-database",
default="slycat",
help="CouchDB database. Default: %(default)s",
)
parser.add_argument(
"--couchdb-host", default="localhost", help="CouchDB host. Default: %(default)s"
)
parser.add_argument(
"--couchdb-port", type=int, default=5984, help="CouchDB port. Default: %(default)s"
)
parser.add_argument(
"--data-store",
default="data-store",
help="Path to the hdf5 data storage directory. Default: %(default)s",
)
parser.add_argument("--force", action="store_true", help="Overwrite existing data.")
parser.add_argument("--project-id", default=[], action="append", help="Project ID to dump. You may specify --project-id multiple times.")
parser.add_argument("--log", default="dumpLog.txt", help="Path to this scripts log file. Default: %(default)s")
parser.add_argument(
"--project-id",
default=[],
action="append",
help="Project ID to dump. You may specify --project-id multiple times.",
)
parser.add_argument(
"--log",
default="dumpLog.txt",
help="Path to this scripts log file. Default: %(default)s",
)
arguments = parser.parse_args()


if os.path.exists(arguments.log):
os.rename(arguments.log, arguments.log+".prev")
os.rename(arguments.log, arguments.log + ".prev")

logging.getLogger().setLevel(logging.INFO)
logging.getLogger().addHandler(logging.FileHandler(arguments.log))
logging.getLogger().handlers[0].setFormatter(logging.Formatter("%(levelname)s - %(message)s"))
#logging.getLogger().handlers[0].setFormatter(logging.Formatter("{} - %(levelname)s - %(message)s".format(sys.argv[0])))
logging.getLogger().handlers[0].setFormatter(
logging.Formatter("%(levelname)s - %(message)s")
)
# logging.getLogger().handlers[0].setFormatter(logging.Formatter("{} - %(levelname)s - %(message)s".format(sys.argv[0])))

if arguments.force and os.path.exists(arguments.output_dir):
shutil.rmtree(arguments.output_dir)
Expand All @@ -44,41 +69,116 @@
os.makedirs(arguments.output_dir)

if arguments.all_projects:
arguments.project_id = set(arguments.project_id + [row["id"] for row in couchdb.view("slycat/projects")])
arguments.project_id = set(
arguments.project_id + [row["id"] for row in couchdb.view("slycat/projects")]
)

if arguments.all_references_bookmarks:
bookmark_ids = list(row["id"] for row in couchdb.view("slycat/project-bookmarks"))
reference_ids = list(row["id"] for row in couchdb.view("slycat/references"))
logging.info("Dumping bookmarks at {0}".format(time.strftime('%X')))
logging.info("Dumping bookmarks at {0}".format(time.strftime("%X")))
for bookmark_id in bookmark_ids:
bookmark = couchdb.get(bookmark_id, attachments=True)
json.dump(bookmark, open(os.path.join(arguments.output_dir, "bookmark-%s.json" % bookmark["_id"]), "w"))
logging.info("Done with bookmarks at {0}".format(time.strftime('%X')))
if len(arguments.project_id) > 0:
for project_id in arguments.project_id:
if bookmark["project"] == project_id:
json.dump(
bookmark,
open(
os.path.join(
arguments.output_dir,
"bookmark-%s.json" % bookmark["_id"],
),
"w",
),
)
else:
json.dump(
bookmark,
open(
os.path.join(
arguments.output_dir, "bookmark-%s.json" % bookmark["_id"]
),
"w",
),
)
logging.info("Done with bookmarks at {0}".format(time.strftime("%X")))

logging.info("Dumping references at {0}".format(time.strftime('%X')))
logging.info("Dumping references at {0}".format(time.strftime("%X")))
for reference_id in reference_ids:
reference = couchdb.get(reference_id, attachments=True)
json.dump(reference, open(os.path.join(arguments.output_dir, "reference-%s.json" % reference["_id"]), "w"))
logging.info("Done with references at {0}".format(time.strftime('%X')))
if len(arguments.project_id) > 0:
for project_id in arguments.project_id:
if reference["project"] == project_id:
json.dump(
reference,
open(
os.path.join(
arguments.output_dir,
"reference-%s.json" % reference["_id"],
),
"w",
),
)
else:
json.dump(
reference,
open(
os.path.join(
arguments.output_dir, "reference-%s.json" % reference["_id"]
),
"w",
),
)
logging.info("Done with references at {0}".format(time.strftime("%X")))

logging.info("Dumping projects at {0}".format(time.strftime('%X')))
logging.info("Dumping projects at {0}".format(time.strftime("%X")))
for project_id in arguments.project_id:
logging.info("Dumping project {0} at {1}".format(project_id, time.strftime('%X')))
logging.info("Dumping project {0} at {1}".format(project_id, time.strftime("%X")))
project = couchdb.get(project_id, attachments=True)
json.dump(project, open(os.path.join(arguments.output_dir, "project-%s.json" % project["_id"]), "w"))
json.dump(
project,
open(
os.path.join(arguments.output_dir, "project-%s.json" % project["_id"]), "w"
),
)
# dump project data
project_datas_ids = list(row["id"] for row in couchdb.view("slycat/project_datas", startkey=project_id, endkey=project_id))
project_datas_ids = list(
row["id"]
for row in couchdb.view(
"slycat/project_datas", startkey=project_id, endkey=project_id
)
)
for project_datas_id in project_datas_ids:
logging.info("Dumping projects-data {0} at {1}".format(project_datas_id, time.strftime('%X')))
logging.info(
"Dumping projects-data {0} at {1}".format(
project_datas_id, time.strftime("%X")
)
)
project_data = couchdb.get(project_datas_id, attachments=True)
json.dump(project_data, open(os.path.join(arguments.output_dir, "projects-data-%s.json" % project_data["_id"]), "w"))

json.dump(
project_data,
open(
os.path.join(
arguments.output_dir, "projects-data-%s.json" % project_data["_id"]
),
"w",
),
)

project_arrays = set()

for row in couchdb.view("slycat/project-models", startkey=project_id, endkey=project_id):
for row in couchdb.view(
"slycat/project-models", startkey=project_id, endkey=project_id
):
logging.info("Dumping model %s", row["id"])
model = couchdb.get(row["id"], attachments=True)
json.dump(model, open(os.path.join(arguments.output_dir, "model-%s.json" % model["_id"]), "w"))
json.dump(
model,
open(
os.path.join(arguments.output_dir, "model-%s.json" % model["_id"]), "w"
),
)

artifact_types = model["artifact-types"]
for key, value in list(model.items()):
Expand All @@ -91,12 +191,22 @@
logging.info("Dumping array set %s", value)
project_arrays.add(value)
try:
shutil.copy(slycat.hdf5.path(value, arguments.data_store),
os.path.join(arguments.output_dir, "array-set-%s.hdf5" % value))
shutil.copy(
slycat.hdf5.path(value, arguments.data_store),
os.path.join(
arguments.output_dir, "array-set-%s.hdf5" % value
),
)
except IOError:
logging.error("is the data store directory correct for hdf5 files? we couldn't find the file")
logging.error(
"is the data store directory correct for hdf5 files? we couldn't find the file"
)
raise Exception("hdf5 file not found")
elif artifact_type in ["file", "json"]:
continue # file artifacts are stored as document attachments, and json artifacts are stored document directly, so we don't have to do anything for them.
else:
logging.warning("Skipping unsupported artifact type: %s %s", artifact_name, artifact_type)
logging.warning(
"Skipping unsupported artifact type: %s %s",
artifact_name,
artifact_type,
)