From bbd72afb26bce89041c484b6e3fb02216d512c53 Mon Sep 17 00:00:00 2001 From: "Patrick W. Crawford" Date: Sun, 1 Aug 2021 18:54:23 -0700 Subject: [PATCH] Updated the readme and including the render restart scripts, and drive appscript utility --- .gitattributes | 1 + README.md | 139 +++++++++++++++++++++++++++++++++---- drive_name.gs | 67 ++++++++++++++++++ render_osx_wrapper.sh | 65 +++++++++++++++++ render_windows_wrapper.bat | 60 ++++++++++++++++ startup.py | 94 +++++++++++++++++++++++++ 6 files changed, 411 insertions(+), 15 deletions(-) create mode 100644 drive_name.gs create mode 100755 render_osx_wrapper.sh create mode 100644 render_windows_wrapper.bat create mode 100644 startup.py diff --git a/.gitattributes b/.gitattributes index 80a1944..6158505 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,2 @@ *.blend filter=lfs diff=lfs merge=lfs -text +*.png filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md index 547e9bd..aa8138c 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,13 @@ This Blender 3d python add-on helps load and render blend files in bulk such as from a community collaboration. -This add-on built and primarily used in Blender 2.90, but may work as far -back as blender 2.80. +This add-on was built and primarily used in Blender 2.90 and 2.93, but may work +as far back as blender 2.80. + +This project was built with scale in mind, to the tune of being able to load, +process, and render 10,000's of blend files. Many of the choices made in +structure and architecture were to improve reliability and consistency, and to +minimize / better handle blender crashes. ## What is this add-on good for? @@ -15,7 +20,7 @@ etc). The key features of this add-on include: -- Quickly load and cycle through user-submitted blend files +- Quickly load and cycle through user-submitted blend files. - Safeguard against any unique user set ups, including blocking python scripts from user files from running (which would be a security concern for the person using this add-on and loading files on their machine). @@ -30,11 +35,15 @@ The key features of this add-on include: other than the sample default use case. - This is because the add-on works best when making specific assumptions about desired inputs and similarly, the desired output. - - Even if you are using custom development time to use this starting point - code, you still end up time vs starting from scratch! + - That being said, it's far better to start with a solid beginning point + and likely you can reuse many of the same utility functions. - Over time, sample branches/releases may be created for different use cases where it has been tailored to meet specific community project needs. +The use case this was built for was to render out an individual pair of frames +(high res and low res) per individual blend file, but the system could be +adjusted for other use cases by adjusting the `process` function as needed. + ## Installing @@ -47,7 +56,7 @@ automatically enabled the next time you start Blender. ### Option 2: One-time load -Instead of installing the script (which ) +Instead of installing the script (especially if continually tweaking), you can temporarily use the add-on without installing it, by dragging and dropping the `blender-community-render.py` file into blender's text editor. Press the play button (or hover over the text and press `alt p`). @@ -56,8 +65,8 @@ button (or hover over the text and press `alt p`). Now that the add-on is enabled, you can find the community render panel under the `Scene` tab of the Properties window. This is the tab where the icon (in -Blender 2.9) looks like an upside-down cone. This panel contains all features of - the add-on. +Blender 2.9) looks like an upside-down cone. This panel contains all features +of the add-on. ### Load form responses @@ -65,10 +74,13 @@ Blender 2.9) looks like an upside-down cone. This panel contains all features of An assumption of this add-on is that user submissions are coming from something like a Google Form (including a file uploader). The add-on looks for a `form_responses.tsv` file, which is a tab-separated-value download of the raw -form responses. +form responses. Tabs are used instead of spaces, toa void issues with escaping +commas in CSV tables, and to avoid using other more verbose formats like json. See the section on preparing this tsv file. This step is really only required -if you are hoping to create in-render text of the author and their country. +if you are hoping to create in-render text of the author and their country, or +if you want to output filenames based on a different field coming from the form +(such as the drive file id itself) instead of the source uploaded filename. While there will be warnings if the tsv file is not found, the addon still works perfectly fine without it. @@ -85,9 +97,17 @@ output folder, you could use something like Drive or Desktop to mount that folder as a network drive. That being said, things will be more stable and generally faster if you are able to directly download the whole folder and save to a custom folder location. In my experience using Drive for Desktop, there is -surprisingly little speed gains to marking the folder offline (assuming files -from users are generally small), copying the whole folder to another local -location will work better and be more stable. +surprisingly little speed gains to marking the folder to be 'available offline', +copying the whole folder to another local location will work better and be more +stable. + +Anecdotally, I also find that loading 10,000's of files to go faster on Mac OSX +than on windows. This is true both on confirming the folder to load, but also +even when simply using blender's built in filebrowser, if navigate into the +folder that has the 10k+ files. Some tricks are used to speed up the loading, +such as caching file listings, but slow is slow. Consider opening a console +window so you can at least monitor the % completion of loading, so you know +it's still working. ### Preparing the form responses @@ -97,10 +117,12 @@ include fields such as (titles/order do not matter here) - Country - The blend file upload - Some kind of affirmation that permission is given to use info + blend +- *Other fields are optional, but could be used to de-duplication repeat entries* To view submissions, you can generate a spreadsheet of responses. Create a new spreadsheet, and then you'll likely want to create a new tab which is separate -from the form responses tab. This is because the field names +from the form responses tab. This is because the field names need to have any +tab characters removed prior to download, to avoid breaking the format. The actual fields expected in the `form_responses.tsv` file are the following (order of columns does not matter here, extra columns are fine too): @@ -109,11 +131,98 @@ The actual fields expected in the `form_responses.tsv` file are the following - blend_filename: Not actually provided by the Google Drive form. Use either an app script or manually join in the name of the blend files based on the drive file url (which *is* provided by the form output). This is used to join an - actual blend file on disk to this row of metadata. + actual blend file on disk to this row of metadata. See the `drive_name.gs` + which was used for this purpose. Final step is to download this form_responses tab as a tsv file, which you can do so from: file > download > Tab-separated values (.tsv, current sheet). +The sheet you download should be in this format: + +![Sample form structure](/sample_form_download.png?raw=true) + +### Using the auto-restart scripts + +**How it's set up** + +Although great effort was put to ensure that blender didn't crash when loading +blend files, it still was inevitable when considering the wide array of blender +versions used for the submitted files. In practice, blender would typically +crash after around 1K blends, but it more had to do with the files submitted. + +There are two categories of crashes relevant here: +1. Random, one-off crashes: Meaning, if the blend file that caused the crash +were opened again, it would be fine. True cause could be due to memory leak or +some other random, non-file related issue. +2. Persistent file issue: There are some blend files that, no matter what, +instant-crash blender when loaded. These should not hold back the rest of the +renders from being done, but rather be logged as a "QC error". + +The idea behind the crash restarting script is that before loading any blend +file, the addon writes a text file to disk. After loading and transforming this +file, the addon deletes this text file. Thus, if the file exists and blender +has closed, it means it crashed (as opposed to a manual / intentional closing +of blender). Thus, the `render_osx_wrapp.sh` or the `render_windows_wrapper.bat` +wrapper scripts check for this file to know whether to attempt to re-open +blender and resume rendering. + +In the two crash categories above, we want to recover and be able to render +(1), but not get stuck in a forever loop when trying to reopen blender to +render (2). Thus, a default of max 3-crashes per blend file is used. That is, +if a blend file causes blender to crash three times, it will permanently skip +it. + +The addon automatically detects even a single crash and logs it as a QC error, +incrementing the number each time, so you can see where files may have happened +to cause a crash but rendered correctly the second time, vs those permanent +failures. + +**Using the startup wrapper** + +1. Firstly, choose the script you are going to use: `render_osx_wrapper.sh` for Mac (Linux should work with at most minor modifications), or `render_windows_wrapper.bat` for windows machines. +2. Update the following paths in the wrapper script: + - blender: The absolute path to the blender executable. + - src_files: The absolute path to the folder containing all the blend files + - render_template: The link to the starting template file, a sample of which + in this repo is [render_template.blend](). + - addon_py: The relative path to the addon script, ie the [community_render.py]() + file, which is used if the addon is not already installed and set up in blender + by default. Note: Due to some loading issues in blender 2.93, it's better + to just install the script as an addon. +3. Make sure the render_template.blend file has been saved where the tsv path + is a valid one to the folder containing the tsv file of form responses. This + will also be the folder where outputs renders are saved (into subfolders). + - To make your life easy, consider just placing the blend file in the same + folder as the tsv file, and ensure the template blend file has the "tsv" + path set to "//", ie the current directory. +4. Open the Terminal (Mac/Linux) or the Command Prompt (Windows), and change into +the folder that contains the `startup.py` script and the according +`render_*_wrapper.*` file. Execute the `render_*_wrapper.*` file. + - The `startup.py` script is used as a script passed into the background + blender process, and pretty much just issues the "resync progress" and + "start render" operations). +5. If you need to end it early, you will need to force quit the script. +Quitting blender itself will, by nature of this restart wrapper, just result in +blender being re-opened a few moments later (also known as "working as +intended"). Force quite typically works by pressing control-c in the terminal +or command prompt window, closing the window, or through the Force Quit Menu +(Mac)/Kill command (Mac/Linux)/Task Manager (Windows). + +## Adjusting the script for other use cases/behavior + +This script is effectively largely because it makes bespoke assumptions about +the use case at hand. This means that, when applied to other projects, the code +should be adjusted. + +The `def process_open_file(context)` function is where you can most easily do +this. This function is called once per blend file when that row is selected, or +during render when it gets to that blend file. This code runs after the entire +blend file has been loaded in as (linked) scene. A simple default use case is +provided with the commented out `process_generic_scene` function. The +donut-rendering use case is covered with the `process_as_donut` function. You +can try creating your own function by modifying either of these, or creating a +new function all together. + ## Contributing Contributions are welcome! See [`CONTRIBUTING.md`](CONTRIBUTING.md) for details. diff --git a/drive_name.gs b/drive_name.gs new file mode 100644 index 0000000..9817e91 --- /dev/null +++ b/drive_name.gs @@ -0,0 +1,67 @@ +/* Populate uploaded filenames from a form given the entry URL + +We need to correlate the row of an entry submission via google form to +the given file that was uploaded. The form response itself only +has the link to the file itself as a column value, not the filename itself. +Connecting the filename to the form row data is necessary for other processes +in the community render system. +*/ + +const _SHEET_NAME = 'tsv_sheet_download'; // Name of the tab to download +const _BATCH_SIZE = 100 // Number of rows to process at once. + + +function populateAllRows(){ + var ss = SpreadsheetApp.getActiveSpreadsheet(); + var targetSheet = ss.getSheetByName(_SHEET_NAME); + Logger.log(targetSheet) + var lastRow = targetSheet.getLastRow(); + Logger.log("Last row:" + lastRow.toString()); + + var updated = 0 + for (var i = 1; i <= Math.floor(lastRow/_BATCH_SIZE); i++) { + var startRow = (i - 1) * _BATCH_SIZE + 1 + var endRow = startRow + _BATCH_SIZE - 1 + if (endRow > lastRow){ + endRow = lastRow + } + updated = updated + populateRow(targetSheet, startRow, endRow); + } + Logger.log("Done, updated " + updated.toString()) +} + +function populateRow(sheet, startRow, endRow) { + const readCol = 5; // E + const writeCol = 6; // F + Logger.log(startRow.toString() +", " + endRow.toString()) + + var readUrls = sheet.getRange(startRow, readCol, endRow-startRow+1, 1).getValues() + var writeRange = sheet.getRange(startRow, writeCol, endRow-startRow+1, 1) + var writeNames = writeRange.getValues() + var updated = 0; + + for (var i = 0; i <= endRow-startRow; i++) { + if (writeNames[i] != ""){ + continue // data already loaded + } + // Logger.log('No data for index, load row '+(startRow+i).toString()); + // all have prefix: https://drive.google.com/open?id= + var id = readUrls[i][0].substring(33, readUrls[i][0].length); + // Logger.log(id) + var this_file = "" + try { + var this_file = DriveApp.getFileById(id) + } + catch(err) { + Logger.log("Failed to pull id "+id.toString()+" from row "+(startRow+i).toString()) + writeNames[i] = [''] + continue + } + writeNames[i] = [this_file.getName()] + updated = updated + 1; + } + // Logger.log("Writing names:"); + // Logger.log(writeNames); + writeRange.setValues(writeNames); + return updated; +} diff --git a/render_osx_wrapper.sh b/render_osx_wrapper.sh new file mode 100755 index 0000000..2017c33 --- /dev/null +++ b/render_osx_wrapper.sh @@ -0,0 +1,65 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# Blender Community Render - OSX Guardian Wrapper +# +# Script used to auto-restart blender in case it crashes in the middle of an +# execution. + +# Hard code this to match your local blender executable location. +blender='/Applications/Blender 2.93/blender.app/Contents/MacOS/blender' + +# Path to where the source blend files are saved +src_files='/source/blender/files_folder' + +# Specify the source blend file to open as the render template +render_template='/path/to/render_tempalte.blend' + +# Assumed the source addon script is in the same directory as active directory. +# Using relative path +addon_py="community_render.py" + +# Only need to make modifications above! + +if [[ ! -f $blender ]] ; then + echo "Blender file missing: $blender". + exit +fi + +# Auto restart blender until THIS script is closed. +echo "Starting blender, will restart until this process is closed." + +restarter="restart_until_finished.txt" +touch $restarter + +while true +do + echo "> Starting blender in 3s (ctrl c to cancel or close window)..." + sleep 3 + "$blender" -b "$render_template" -P startup.py -- \ + -src_files "$src_files" \ + -addon_py "$(pwd)/$addon_py" + echo "Blender exited" + # exit + + if [[ ! -f $restarter ]] ; then + echo "Restarter file is missing, assumign render completed.". + exit + fi + +done diff --git a/render_windows_wrapper.bat b/render_windows_wrapper.bat new file mode 100644 index 0000000..0bdcfa6 --- /dev/null +++ b/render_windows_wrapper.bat @@ -0,0 +1,60 @@ +:: License: GPL v3 +@echo off + +:: Assign the explicit version of blender to use, full path +set blender="C:\path\blender\blender.exe" + +:: Full path to where the source blend files are saved +set src_files="D:\blender\file\submissions" + +:: Specify the source blend to open as the render template +set render_template="G:\working_directory\render_template.blend" + +:: Specify the addon itself, should be the SAME directory as this script runs from +set addon_py="G:\working_directory\community_render.py" + +:: Only need to make modifications above! + +:: Create the file for indicating renders are in progress +set restarter="restart_until_finished.txt" +break>%restarter% + +if not exist %blender% ( + echo "Blender file is missing, exiting" + goto :eof +) + +if not exist %src_files% ( + echo "Source files missing, exiting" + goto :eof +) + +if not exist %render_template% ( + echo "Render template missing missing, exiting" + goto :eof +) + +if not exist %addon_py% ( + echo "addon_py file missing missing, exiting" + echo %addon_py% + goto :eof +) + +if not exist %restarter% ( + echo "Restarter file doesn't exist, won't ever close" + echo %restarter% + goto :eof +) + +:: Change directory to another drive using cd /d "G:\etc" +:startloop +%blender% -b %render_template% -P startup.py -- -src_files %src_files% -addon_py %addon_py% +echo "Blender exited" + +if not exist %restarter% ( + echo "Restarter file doesn't exist, render completed!" + goto :eof +) + +goto startloop +@echo on diff --git a/startup.py b/startup.py new file mode 100644 index 0000000..10e92dd --- /dev/null +++ b/startup.py @@ -0,0 +1,94 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +"""Script to load and run the main community script, and then start render. + +Meant to be used with the {platform}_wrapper executable, to auto-restart +blender if/when it crashes. +""" + +import os +import sys + +import bpy + + +if __name__ == '__main__': + args = (sys.argv[sys.argv.index("--") + 1:] if "--" in sys.argv else []) + print("Launching scirpt with args: ", args) + + # Expecting to get flags for: + if "-src_files" in args: + ind = args.index("-src_files") + 1 + print("Src path:", args[ind]) + src_files = args[ind] + else: + src_files = None + + if "-addon_py" in args: + ind = args.index("-addon_py") + 1 + print("Addon code:", args[ind]) + addon_py = args[ind] + else: + addon_py = None + + if not addon_py or not src_files: + print(f"Missing addon_py ({addon_py}) or blends ({src_files})") + sys.exit() + + # Attempt to load the addon from disk if not already installed and enabled. + if "crp_props" not in dir(bpy.context.scene): + print("Registering addon...") + text = bpy.data.texts.load(addon_py) + mod = text.as_module() + try: + mod.register() + except Exception as err: + # In blender 2.93, it seems to fail due to some typing/annotation + # issues, even though it runs fine when run through the UI directly. + # Likely due to how Blender has implemented annotations, being not + # actual python standard. + print(err) + + # This method apparently matches the built-in run_script operator. + print("Attempting to recover from mod register error using exec:") + exec(compile(open(addon_py).read(), addon_py, 'exec')) + else: + print("Addon alrady enabled") + # Seems we need to have addon already enabled, otherwise the load-text + # and register seems to not properly register operators for calls. + + print("Community code: Loading files...") + props = bpy.context.scene.crp_props + props.config_folder = os.getcwd() # For form and setting output location. + if src_files[-1] != os.path.sep: + src_files += os.path.sep + props.source_folder = src_files # Will trigger reload of all blend files. + + if not props.file_list: + print("No rows loaded, exiting") + sys.exit() + + print("Starting render!") + # bpy.ops.crp.render_all_interactive() # Interactive won't work, as a modal. + bpy.ops.crp.render_all_files() + + # Try to remove the file which keeps blender restarting. + print("Renders finished!") + os.remove("restart_until_finished.txt") + # sys.exit() # Un-comment to auto-close blender when done rendering.