diff --git a/Makefile b/Makefile index 9379b84bd..1e5119152 100755 --- a/Makefile +++ b/Makefile @@ -777,6 +777,8 @@ ifeq ($(TARGET_SWITCH),1) endif C_DEFINES := $(foreach d,$(DEFINES),-D$(d)) +C_DEFINES += $(CUSTOM_C_DEFINES) + DEF_INC_CFLAGS := $(foreach i,$(INCLUDE_DIRS),-I $(i)) $(C_DEFINES) # Set C Preprocessor flags @@ -786,7 +788,7 @@ else ifeq ($(COMPILER_TYPE),clang) CPPFLAGS := -E -P -x c -Wno-trigraphs endif -CPPFLAGS += $(DEF_INC_CFLAGS) $(CUSTOM_C_DEFINES) +CPPFLAGS += $(DEF_INC_CFLAGS) ASMDEFINES := $(VER_DEFINES) $(GRU_DEFINES) $(ULTRA_VER_DEF) @@ -857,6 +859,7 @@ ifeq ($(CPP_ASSEMBLY),1) ASFLAGS := -march=vr4300 -mabi=32 $(foreach i,$(INCLUDE_DIRS),-I$(i)) $(foreach d,$(ASMDEFINES),--defsym $(d)) else ASMFLAGS := -G 0 $(DEF_INC_CFLAGS) -w -nostdinc -c -march=vr4300 -mfix4300 -mno-abicalls -DMIPSEB -D_LANGUAGE_ASSEMBLY -D_MIPS_SIM=1 -D_MIPS_SZLONG=32 + ASMFLAGS += $(CUSTOM_C_DEFINES) endif RSPASMFLAGS := $(foreach d,$(ASMDEFINES),-definelabel $(subst =, ,$(d))) @@ -865,9 +868,6 @@ OBJCOPYFLAGS := --pad-to=0x800000 --gap-fill=0xFF SYMBOL_LINKING_FLAGS := --no-check-sections $(addprefix -R ,$(SEG_FILES)) LDFLAGS := -T $(BUILD_DIR)/$(LD_SCRIPT) -Map $(BUILD_DIR)/sm64.$(VERSION).map $(SYMBOL_LINKING_FLAGS) -CFLAGS += $(CUSTOM_C_DEFINES) -ASMFLAGS += $(CUSTOM_C_DEFINES) - else # TARGET_N64 ifeq ($(TARGET_WII_U),1) @@ -1087,11 +1087,9 @@ ifeq ($(CPP_ASSEMBLY),1) endif else ASMFLAGS := $(DEF_INC_CFLAGS) -D_LANGUAGE_ASSEMBLY + ASMFLAGS += $(CUSTOM_C_DEFINES) endif -CFLAGS += $(CUSTOM_C_DEFINES) -ASMFLAGS += $(CUSTOM_C_DEFINES) - # Load external textures ifeq ($(EXTERNAL_DATA),1) CFLAGS += -DFS_BASEDIR="\"$(BASEDIR)\"" @@ -1250,7 +1248,6 @@ endif ifeq ($(EXTERNAL_DATA),1) BASEPACK_PATH := $(BUILD_DIR)/$(BASEDIR)/$(BASEPACK) -BASEPACK_LST := $(BUILD_DIR)/basepack.lst # depend on resources as well all: $(BASEPACK_PATH) @@ -1258,32 +1255,10 @@ all: $(BASEPACK_PATH) # phony target for building resources res: $(BASEPACK_PATH) -# prepares the basepack.lst -$(BASEPACK_LST): $(EXE_DEPEND) - @$(PRINT) "$(GREEN)Generating external data list: $(BLUE)$@ $(NO_COL)\n" - @mkdir -p $(BUILD_DIR)/$(BASEDIR) - @echo "$(BUILD_DIR)/sound/bank_sets sound/bank_sets" > $(BASEPACK_LST) - @echo "$(BUILD_DIR)/sound/sequences.bin sound/sequences.bin" >> $(BASEPACK_LST) - @echo "$(BUILD_DIR)/sound/sound_data.ctl sound/sound_data.ctl" >> $(BASEPACK_LST) - @echo "$(BUILD_DIR)/sound/sound_data.tbl sound/sound_data.tbl" >> $(BASEPACK_LST) - ifeq ($(VERSION),sh) - @echo "$(BUILD_DIR)/sound/sequences_header sound/sequences_header" >> $(BASEPACK_LST) - @echo "$(BUILD_DIR)/sound/ctl_header sound/ctl_header" >> $(BASEPACK_LST) - @echo "$(BUILD_DIR)/sound/tbl_header sound/tbl_header" >> $(BASEPACK_LST) - endif - @$(foreach f, $(wildcard $(SKYTILE_DIR)/*), echo $(f) gfx/$(f:$(BUILD_DIR)/%=%) >> $(BASEPACK_LST);) - @find actors -name \*.png -exec echo "{} gfx/{}" >> $(BASEPACK_LST) \; - @find levels -name \*.png -exec echo "{} gfx/{}" >> $(BASEPACK_LST) \; - @find textures -name \*.png -exec echo "{} gfx/{}" >> $(BASEPACK_LST) \; - ifeq ($(PORT_MOP_OBJS),1) - @find src/extras/mop/actors -name \*.png -exec echo "{} gfx/{}" >> $(BASEPACK_LST) \; - endif - -# prepares the resource ZIP with base data -$(BASEPACK_PATH): $(BASEPACK_LST) - $(call print,Zipping from list:,$<,$@) - $(V)$(PYTHON) $(TOOLS_DIR)/mkzip.py $(BASEPACK_LST) $(BASEPACK_PATH) - +# Combined generation and packing +$(BASEPACK_PATH): $(EXE_DEPEND) + @$(PRINT) "$(GREEN)Packing external data list: $(BLUE)$@ $(NO_COL)\n" + $(V)$(PYTHON) pack_extdata.py --build-dir "$(BUILD_DIR)" --skytile-dir "$(SKYTILE_DIR)" --output "$@" $(C_DEFINES) > /dev/null endif clean: @@ -1453,6 +1428,15 @@ $(BUILD_DIR)/%: %.png $(BUILD_DIR)/%.inc.c: $(BUILD_DIR)/% %.png $(call print,Converting:,$<,$@) $(V)hexdump -v -e '1/1 "0x%X,"' $< > $@ + +# Store crash screen textures so they render even without any texture data +$(BUILD_DIR)/$(TEXTURE_DIR)/crash_screen_pc/%.inc.c: $(TEXTURE_DIR)/crash_screen_pc/%.png + $(call print,Converting crash textures:,$<,$@) + $(V)$(N64GRAPHICS) -s $(TEXTURE_ENCODING) -i $@ -g $< -f $(lastword $(subst ., ,$(basename $<))) + +$(BUILD_DIR)/$(TEXTURE_DIR)/crash_screen_pc/%: $(TEXTURE_DIR)/crash_screen_pc/%.png + $(call print,Converting crash textures:,$<,$@) + $(V)$(N64GRAPHICS) -s raw -i $@ -g $< -f $(lastword $(subst ., ,$@)) else $(BUILD_DIR)/%: %.png $(call print,Converting:,$<,$@) diff --git a/pack_extdata.py b/pack_extdata.py new file mode 100644 index 000000000..7ad81ba98 --- /dev/null +++ b/pack_extdata.py @@ -0,0 +1,543 @@ +#!/usr/bin/env python3 + +import os +import sys +import glob +import zipfile +import hashlib +import json +import multiprocessing +from concurrent.futures import ThreadPoolExecutor, as_completed +import time +import argparse + +def parse_defines(defines_list): + """Parse -D arguments into a dictionary of defines (C-style)""" + defines = {} + for define in defines_list: + if define.startswith('-D'): + define = define[2:] # Remove the -D prefix + + # Handle both KEY and KEY=VALUE formats + if '=' in define: + key, value = define.split('=', 1) + defines[key] = value + else: + defines[define] = True # Flag-style define (no value) + + return defines + +def is_defined(defines, key): + """Check if a define exists (C-style: any value means defined)""" + return key in defines + +def calculate_file_hash(filepath): + """Calculate MD5 hash of a file for caching""" + hash_md5 = hashlib.md5() + try: + with open(filepath, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + return hash_md5.hexdigest() + except (IOError, OSError): + return None + +def generate_file_list(build_dir, defines, skytile_dir): + """Generate the list of files to include in the basepack""" + file_list = [] + + # Sound files + sound_files = [ + ('bank_sets', 'sound/bank_sets'), + ('sequences.bin', 'sound/sequences.bin'), + ('sound_data.ctl', 'sound/sound_data.ctl'), + ('sound_data.tbl', 'sound/sound_data.tbl') + ] + + # SH/CN version specific files + if is_defined(defines, 'VERSION_SH') or is_defined(defines, 'VERSION_CN'): + sound_files.extend([ + ('sequences_header', 'sound/sequences_header'), + ('ctl_header', 'sound/ctl_header'), + ('tbl_header', 'sound/tbl_header') + ]) + + for filename, archive_path in sound_files: + real_path = os.path.join(build_dir, 'sound', filename) + if os.path.exists(real_path): + file_list.append((real_path, archive_path)) + + # Skybox tiles + if skytile_dir and os.path.exists(skytile_dir): + for tile_file in glob.glob(os.path.join(skytile_dir, '*')): + if os.path.isfile(tile_file): + archive_path = f"gfx/{os.path.relpath(tile_file, build_dir)}" + file_list.append((tile_file, archive_path)) + + # PNG files in various directories + folders_to_search = ['actors', 'levels', 'textures'] + + if is_defined(defines, 'PORT_MOP_OBJS'): + folders_to_search.append('src/extras/mop/actors') + + # Exact directory paths to exclude + exclude_paths = ['textures/crash_screen', 'textures/crash_screen_pc'] + + if not is_defined(defines, 'VERSION_CN'): + exclude_paths.append('textures/segment2/cn') + + # Normalize all paths at once + exclude_list = [os.path.normpath(path) for path in exclude_paths] + + for folder in folders_to_search: + if os.path.exists(folder): + for root, dirs, files in os.walk(folder): + # Check if current directory should be excluded (exact match) + normalized_root = os.path.normpath(root) + should_exclude = normalized_root in exclude_list + + if should_exclude: + # Skip this directory and all its subdirectories + continue + + for file in files: + if file.endswith('.png'): + real_path = os.path.join(root, file) + archive_path = f"gfx/{real_path}" + file_list.append((real_path, archive_path)) + + return file_list + +def load_ndjson_cache(cache_file): + """Load NDJSON cache file""" + cache = {} + if os.path.exists(cache_file): + try: + with open(cache_file, 'r') as f: + for line in f: + line = line.strip() + if line: + try: + entry = json.loads(line) + cache_key = entry.get('key') + if cache_key: + cache[cache_key] = entry + except json.JSONDecodeError: + continue + except IOError: + pass + return cache + +def save_ndjson_cache(cache_file, cache_data): + """Save cache as NDJSON (faster for large caches)""" + try: + with open(cache_file, 'w') as f: + for cache_key, data in cache_data.items(): + entry = {'key': cache_key, **data} + f.write(json.dumps(entry) + '\n') + except IOError: + pass + +def clean_cache(build_dir): + """Clean the cache file""" + cache_file = os.path.join(build_dir, 'basepack_cache.ndjson') + + if os.path.exists(cache_file): + try: + os.remove(cache_file) + print(f"Removed cache file: {cache_file}") + return True + except OSError as e: + print(f"Error removing cache file: {e}") + return False + else: + print("Cache file does not exist") + return True + +def get_files_to_pack(file_list, cache_file): + """Get list of files that need to be packed (based on cache)""" + old_cache = load_ndjson_cache(cache_file) + files_to_pack = [] + + # Get all files that should be in the current build + current_files = {real_path: archive_path for real_path, archive_path in file_list} + + # Clean up cache by removing entries for files that: + # 1. No longer exist on filesystem, OR + # 2. Should not be included in current build (due to define conditions) + cleaned_cache = {} + for cache_key, cache_data in old_cache.items(): + if ':' in cache_key: + real_path = cache_key.split(':', 1)[0] + + # Keep in cache only if: + # 1. File exists on filesystem, AND + # 2. File should be included in current build + if os.path.exists(real_path) and real_path in current_files: + cleaned_cache[cache_key] = cache_data + + # Now check which files need to be packed + for real_path, archive_path in file_list: + if not os.path.exists(real_path): + continue + + file_hash = calculate_file_hash(real_path) + if file_hash is None: + continue + + mtime = os.path.getmtime(real_path) + cache_key = f"{real_path}:{archive_path}" + + # Check if file has changed or is new + if (cache_key in cleaned_cache and + cleaned_cache[cache_key].get('hash') == file_hash and + cleaned_cache[cache_key].get('mtime') == mtime): + # File hasn't changed, use cached version + pass + else: + # File has changed or is new + files_to_pack.append((real_path, archive_path)) + # Update cache entry for this file + cleaned_cache[cache_key] = { + 'hash': file_hash, + 'mtime': mtime, + 'size': os.path.getsize(real_path) + } + + return files_to_pack, cleaned_cache + +def prepare_file_for_zip(real_path, archive_path): + """Read file content and prepare it for ZIP writing (thread-safe)""" + try: + with open(real_path, 'rb') as f: + content = f.read() + return True, real_path, archive_path, content, None + except (IOError, OSError) as e: + return False, real_path, archive_path, None, str(e) + +def print_progress(current, total, start_time, message=""): + """Simple progress indicator with ASCII-only characters""" + if total == 0: + return + + elapsed = time.time() - start_time + percent = (current / total) * 100 + bar_length = 20 + filled_length = int(bar_length * current // total) + + bar = '=' * filled_length + '-' * (bar_length - filled_length) + + if current > 0: + time_per_file = elapsed / current + remaining = time_per_file * (total - current) + time_info = f"{elapsed:.1f}s elapsed, {remaining:.1f}s remaining" + else: + time_info = f"{elapsed:.1f}s elapsed" + + progress_text = f"\r{message} |{bar}| {current}/{total} ({percent:.1f}%) {time_info}" + + try: + sys.stdout.write(progress_text) + sys.stdout.flush() + except UnicodeEncodeError: + # Fallback: simpler progress without Unicode + simple_text = f"\r{message} {current}/{total} ({percent:.1f}%) {time_info}" + sys.stdout.write(simple_text) + sys.stdout.flush() + +def verify_zip_contents(zip_path, expected_files, changed_files=None): + """Verify that the ZIP contains the expected files and check changed files""" + if not os.path.exists(zip_path): + print(f"ERROR: ZIP file {zip_path} does not exist") + return False + + try: + with zipfile.ZipFile(zip_path, 'r') as zipf: + zip_files = set(zipf.namelist()) + + # Check if all expected files are present + missing_files = expected_files - zip_files + if missing_files: + print(f"ERROR: {len(missing_files)} files missing from ZIP:") + for missing in sorted(missing_files)[:10]: # Show first 10 missing + print(f" - {missing}") + if len(missing_files) > 10: + print(f" ... and {len(missing_files) - 10} more") + return False + + # Check if any unexpected files are present + extra_files = zip_files - expected_files + if extra_files: + print(f"WARNING: {len(extra_files)} unexpected files in ZIP:") + for extra in sorted(extra_files)[:5]: # Show first 5 extra + print(f" - {extra}") + if len(extra_files) > 5: + print(f" ... and {len(extra_files) - 5} more") + + # Verify changed files if provided + if changed_files: + verified_changes = 0 + failed_changes = 0 + + for real_path, archive_path in changed_files: + if archive_path in zip_files: + # Calculate hash of the file in ZIP + with zipf.open(archive_path) as zipped_file: + zip_content = zipped_file.read() + zip_hash = hashlib.md5(zip_content).hexdigest() + + # Calculate hash of the original file + file_hash = calculate_file_hash(real_path) + + if file_hash and zip_hash == file_hash: + verified_changes += 1 + else: + failed_changes += 1 + print(f"VERIFY FAIL: {archive_path} - hash mismatch") + else: + failed_changes += 1 + print(f"VERIFY FAIL: {archive_path} - missing from ZIP") + + if failed_changes > 0: + print(f"File verification: {verified_changes} OK, {failed_changes} FAILED") + return False + else: + print(f"File verification: All {verified_changes} changed files verified OK") + + return True + + except (IOError, zipfile.BadZipFile) as e: + print(f"ERROR: Failed to verify ZIP file: {e}") + return False + +def create_basepack(build_dir, defines, skytile_dir, output_zip, num_workers): + """Generate file list and create basepack ZIP with optimizations""" + cache_file = os.path.join(build_dir, 'basepack_cache.ndjson') + + # Generate complete file list + print("Scanning for files...") + file_list = generate_file_list(build_dir, defines, skytile_dir) + print(f"Found {len(file_list)} total files") + + # Check which files need to be packed + print("Checking cache for changes...") + files_to_pack, cleaned_cache = get_files_to_pack(file_list, cache_file) + + # Track changed files for verification + changed_files = files_to_pack.copy() + + # Get the set of archive paths that should be in the final ZIP + current_archive_paths = {archive_path for _, archive_path in file_list} + + # Check if ZIP needs to be rebuilt to remove unwanted files + need_cleanup = False + if os.path.exists(output_zip): + try: + with zipfile.ZipFile(output_zip, 'r') as old_zip: + current_zip_files = set(old_zip.namelist()) + # Check if there are files in ZIP that shouldn't be there + unwanted_files = current_zip_files - current_archive_paths + if unwanted_files: + print(f"Found {len(unwanted_files)} unwanted files in ZIP that need removal") + need_cleanup = True + except (IOError, zipfile.BadZipFile): + # ZIP is corrupt or can't be read, need to rebuild + need_cleanup = True + + # We need to rebuild if: + # 1. There are files to pack (changed/new files), OR + # 2. There are unwanted files in the ZIP that need removal + # 3. Cache file doesn't exist but ZIP does (force rebuild to avoid duplicates) + cache_exists = os.path.exists(cache_file) + need_rebuild = bool(files_to_pack) or need_cleanup or (os.path.exists(output_zip) and not cache_exists) + + if not need_rebuild: + print("No files need to be updated in basepack") + # Still save the cleaned cache (in case files were removed from filesystem) + save_ndjson_cache(cache_file, cleaned_cache) + return True + + if need_cleanup and not files_to_pack: + print("Cleaning up unwanted files from ZIP...") + elif not cache_exists and os.path.exists(output_zip): + print("Cache missing, rebuilding ZIP to avoid duplicates...") + + if files_to_pack: + print(f"Processing {len(files_to_pack)} changed files:") + for real_path, archive_path in files_to_pack: + print(f" - {archive_path}") + + # Use ThreadPoolExecutor for parallel file reading + start_time = time.time() + successful_reads = 0 + failed_reads = 0 + failed_files = [] + file_contents = [] + + if files_to_pack: + with ThreadPoolExecutor(max_workers=num_workers) as executor: + # Submit all file reading tasks + future_to_file = { + executor.submit(prepare_file_for_zip, real_path, archive_path): (real_path, archive_path) + for real_path, archive_path in files_to_pack + } + + # Process results with progress indicator + completed = 0 + total = len(files_to_pack) + + for future in as_completed(future_to_file): + real_path, archive_path = future_to_file[future] + try: + success, file_path, arch_path, content, error = future.result() + if success: + successful_reads += 1 + file_contents.append((arch_path, content)) + else: + failed_reads += 1 + failed_files.append((file_path, error)) + except Exception as e: + failed_reads += 1 + failed_files.append((real_path, str(e))) + + completed += 1 + print_progress(completed, total, start_time, "Reading files") + + # Print newline to clear the progress bar + print() + if files_to_pack: + print(f"File reading complete: {successful_reads} successful, {failed_reads} failed") + + if failed_reads > 0: + print("\nFailed to read files:") + for file_path, error in failed_files: + print(f" - {file_path}: {error}") + + # Now create the ZIP file + if need_rebuild: + print(f"Creating ZIP archive...") + try: + # Create a temporary file first + temp_zip = output_zip + '.tmp' + + # Create a mapping of archive paths to content for changed files + changed_files_map = {archive_path: content for archive_path, content in file_contents} + + with zipfile.ZipFile(temp_zip, 'w', zipfile.ZIP_DEFLATED, allowZip64=False) as zipf: + added_files = set() + + # First, add all files from the old ZIP that should remain (excluding changed ones) + if os.path.exists(output_zip): + try: + with zipfile.ZipFile(output_zip, 'r') as old_zip: + for old_info in old_zip.infolist(): + if (old_info.filename in current_archive_paths and + old_info.filename not in changed_files_map and + old_info.filename not in added_files): + + # This file should remain and hasn't been changed + with old_zip.open(old_info) as old_file: + content = old_file.read() + zipf.writestr(old_info.filename, content) + added_files.add(old_info.filename) + if len(added_files) % 100 == 0: + print_progress(len(added_files), len(current_archive_paths), start_time, "Copying unchanged files") + except (IOError, zipfile.BadZipFile): + # Old ZIP is corrupt, start fresh + pass + + # Now add all changed files (this will overwrite any existing ones) + for i, (archive_path, content) in enumerate(file_contents): + zipf.writestr(archive_path, content) + added_files.add(archive_path) + + if len(added_files) % 100 == 0 or i + 1 == len(file_contents): + print_progress(len(added_files), len(current_archive_paths), start_time, "Adding changed files") + + # Replace the old ZIP with the new one + if os.path.exists(output_zip): + os.remove(output_zip) + os.rename(temp_zip, output_zip) + + # Print newline to clear the progress bar + print() + + # Verify the ZIP contents + print("Verifying ZIP contents...") + verification_success = verify_zip_contents(output_zip, current_archive_paths, changed_files if files_to_pack else None) + + if not verification_success: + print("ERROR: ZIP verification failed!") + return False + + # Update cache AFTER successful ZIP creation and verification + print("Updating cache...") + # The cache is already updated with new/changed files in get_files_to_pack + # We just need to save it + save_ndjson_cache(cache_file, cleaned_cache) + + final_count = len(added_files) + print(f"Successfully created and verified ZIP with {final_count} files") + return True + + except (IOError, OSError, zipfile.BadZipFile) as e: + print(f"\nError creating ZIP file: {e}") + # Clean up temporary file if it exists + if os.path.exists(temp_zip): + os.remove(temp_zip) + return False + else: + print("No files to write to ZIP") + # Still save the cleaned cache (in case files were removed from filesystem) + save_ndjson_cache(cache_file, cleaned_cache) + return failed_reads == 0 + +def main(): + parser = argparse.ArgumentParser(description='EXTERNAL_DATA zip packer for sm64ex') + parser.add_argument('--build-dir', required=True, help='Build directory') + parser.add_argument('--skytile-dir', help='Skybox tiles directory') + parser.add_argument('--output', help='Output ZIP file') + parser.add_argument('--workers', type=int, help='Number of worker threads (default: CPU count - 1)') + parser.add_argument('--clean', action='store_true', help='Clean cache file and exit') + parser.add_argument('-D', action='append', default=[], help='Define C flags') + + args = parser.parse_args() + + # Handle clean operation + if args.clean: + success = clean_cache(args.build_dir) + sys.exit(0 if success else 1) + + # Validate required arguments for non-clean operations + if not args.output: + parser.error("the following arguments are required for packing: --output") + + # Parse defines (C-style: both KEY and KEY=VALUE) + defines = parse_defines(args.D) + + # Determine number of workers + if args.workers is not None: + num_workers = max(1, args.workers) + else: + num_workers = max(1, multiprocessing.cpu_count() - 1) + + # Create output directory + os.makedirs(os.path.dirname(args.output), exist_ok=True) + + success = create_basepack( + args.build_dir, + defines, + args.skytile_dir, + args.output, + num_workers + ) + + if success: + print(f"Successfully created: {args.output}") + sys.exit(0) + else: + print(f"Failed to create: {args.output}") + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/src/pc/configfile.c b/src/pc/configfile.c index 7a49b9249..9abe5ddcc 100644 --- a/src/pc/configfile.c +++ b/src/pc/configfile.c @@ -93,8 +93,8 @@ unsigned int configRumbleStrength = 50; bool configAutohideTouch = false; #endif -#ifdef EXTERNAL_DATA -bool configPrecacheRes = true; +#if defined(EXTERNAL_DATA) && !defined(TARGET_PORT_CONSOLE) +bool configPrecacheRes = false; #endif #ifdef MOUSE_ACTIONS @@ -174,9 +174,9 @@ static const struct ConfigOption options[] = { {.name = "stick_deadzone", .type = CONFIG_TYPE_UINT, .uintValue = &configStickDeadzone}, {.name = "rumble_strength", .type = CONFIG_TYPE_UINT, .uintValue = &configRumbleStrength}, #endif - #ifdef EXTERNAL_DATA +#if defined(EXTERNAL_DATA) && !defined(TARGET_PORT_CONSOLE) {.name = "precache", .type = CONFIG_TYPE_BOOL, .boolValue = &configPrecacheRes}, - #endif +#endif #ifdef MOUSE_ACTIONS {.name = "mouse_enable", .type = CONFIG_TYPE_BOOL, .boolValue = &configMouse}, #endif diff --git a/src/pc/gfx/gfx_pc.c b/src/pc/gfx/gfx_pc.c index 250cc24d4..d28bfe159 100644 --- a/src/pc/gfx/gfx_pc.c +++ b/src/pc/gfx/gfx_pc.c @@ -210,6 +210,37 @@ static inline size_t string_hash(const uint8_t *str) { h = 31 * h + *p; return h; } + +// Check for png header +static inline int is_png_file(const char *filename) { + fs_file_t *file = fs_open(filename); + if (!file) return 0; + + unsigned char header[8]; + size_t bytes_read = fs_read(file, header, 8); + fs_close(file); + + return (bytes_read == 8 && + header[0] == 0x89 && header[1] == 0x50 && + header[2] == 0x4E && header[3] == 0x47 && + header[4] == 0x0D && header[5] == 0x0A && + header[6] == 0x1A && header[7] == 0x0A); +} + +static inline int is_file_texture(const char *texture_data) { + // Quick check: if it contains path separators or extensions, it's likely a file + if (strchr(texture_data, '/') || strchr(texture_data, '\\') || + strchr(texture_data, '.') || strlen(texture_data) < SYS_MAX_PATH) { + + char test_path[SYS_MAX_PATH]; + snprintf(test_path, sizeof(test_path), FS_TEXTUREDIR "/%s.png", texture_data); + + if (is_png_file(test_path)) { + return 1; + } + } + return 0; +} #endif #ifdef TARGET_N3DS @@ -330,14 +361,14 @@ static struct ColorCombiner *gfx_lookup_or_create_color_combiner(uint32_t cc_id) } static bool gfx_texture_cache_lookup(int tile, struct TextureHashmapNode **n, const uint8_t *orig_addr, uint32_t fmt, uint32_t siz, const uint8_t *palette, uint32_t checksum) { - - #ifdef EXTERNAL_DATA // hash and compare the data (i.e. the texture name) itself - size_t hash = string_hash(orig_addr); - #define CMPADDR(x, y) (x && !sys_strcasecmp((const char *)x, (const char *)y)) - #else // hash and compare the address +#ifdef EXTERNAL_DATA // hash and compare the texture name data if it exists, otherwise use the address + int is_ext_img = is_file_texture((const char*)orig_addr); + size_t hash = is_ext_img ? string_hash(orig_addr) : (uintptr_t)orig_addr; + #define CMPADDR(x, y) (is_ext_img ? (x && !sys_strcasecmp((const char *)x, (const char *)y)) : (x == y)) +#else // hash and compare the address size_t hash = (uintptr_t)orig_addr; #define CMPADDR(x, y) x == y - #endif +#endif hash = (hash >> HASH_SHIFT) & HASH_MASK; @@ -376,8 +407,6 @@ static bool gfx_texture_cache_lookup(int tile, struct TextureHashmapNode **n, co #undef CMPADDR } -#ifndef EXTERNAL_DATA - static uint8_t rgba32_buf[32768] __attribute__((aligned(32))); static void import_texture_rgba16(int tile) { @@ -579,22 +608,46 @@ static void import_texture_ci8(int tile) { gfx_rapi->upload_texture(rgba32_buf, width, height); } -#else // EXTERNAL_DATA +#ifdef EXTERNAL_DATA +static int fs_stb_read(void *user, char *data, int size) { + fs_file_t *file = (fs_file_t *)user; + return (int)fs_read(file, data, size); +} + +static void fs_stb_skip(void *user, int n) { + fs_file_t *file = (fs_file_t *)user; + fs_seek(file, fs_tell(file) + n); +} + +static int fs_stb_eof(void *user) { + fs_file_t *file = (fs_file_t *)user; + return fs_eof(file); +} static inline void load_texture(const char *fullpath) { - int w, h; - uint64_t imgsize = 0; - - u8 *imgdata = fs_load_file(fullpath, &imgsize); - if (imgdata) { - // TODO: implement stbi_callbacks or something instead of loading the whole texture - u8 *data = stbi_load_from_memory(imgdata, imgsize, &w, &h, NULL, 4); - free(imgdata); - if (data) { - gfx_rapi->upload_texture(data, w, h); - stbi_image_free(data); // don't need this anymore - return; - } + int w, h, c; + fs_file_t *file = fs_open(fullpath); + + if (!file) { + fprintf(stderr, "could not open texture: `%s`\n", fullpath); + // replace with missing texture + gfx_rapi->upload_texture(missing_texture, MISSING_W, MISSING_H); + return; + } + + stbi_io_callbacks callbacks = { + .read = fs_stb_read, + .skip = fs_stb_skip, + .eof = fs_stb_eof, + }; + + u8 *data = stbi_load_from_callbacks(&callbacks, file, &w, &h, &c, 4); + fs_close(file); + + if (data) { + gfx_rapi->upload_texture(data, w, h); + stbi_image_free(data); // don't need this anymore + return; } fprintf(stderr, "could not load texture: `%s`\n", fullpath); @@ -602,8 +655,7 @@ static inline void load_texture(const char *fullpath) { gfx_rapi->upload_texture(missing_texture, MISSING_W, MISSING_H); } - -// this is taken straight from n64graphics +#ifndef TARGET_PORT_CONSOLE static bool texname_to_texformat(const char *name, u8 *fmt, u8 *siz) { static const struct { const char *name; @@ -648,26 +700,45 @@ static bool preload_texture(void *user, const char *path) { char *dot = strrchr(texname, '.'); if (dot) *dot = 0; - // get the format and size from filename + // print current file being cached + #ifdef _WIN32 + // use spaces method + static int last_len = 0; + if (last_len > 0) { + fprintf(stdout, "\r%*s\r", last_len, ""); + } + last_len = fprintf(stdout, "precaching: %s", path); + #else + // use ANSI codes + fprintf(stdout, "\033[2K\rprecaching: %s", path); + #endif + fflush(stdout); + + // Get the format and size from filename u8 fmt, siz; if (!texname_to_texformat(texname, &fmt, &siz)) { fprintf(stderr, "unknown texture format: `%s`, skipping\n", texname); return true; // just skip it, might be a stray skybox or something } - char *actualname = texname; - // strip off the prefix // TODO: make a fs_ function for this shit - if (!strncmp(FS_TEXTUREDIR "/", actualname, 4)) actualname += 4; + const char *actualname = texname; + const char *prefix = FS_TEXTUREDIR "/"; // strip off the "gfx/" prefix + size_t prefix_len = strlen(prefix); + if (!strncmp(actualname, prefix, prefix_len)) { + actualname += prefix_len; + } // this will be stored in the hashtable, so make a copy actualname = sys_strdup(actualname); assert(actualname); struct TextureHashmapNode *n; - if (!gfx_texture_cache_lookup(0, &n, actualname, fmt, siz, 0, 0)) + if (!gfx_texture_cache_lookup(0, &n, actualname, fmt, siz, 0, 0)) { load_texture(path); // new texture, load it + } return true; } +#endif // TARGET_PORT_CONSOLE #endif // EXTERNAL_DATA @@ -740,10 +811,13 @@ static void import_texture(int tile) { #ifdef EXTERNAL_DATA // the "texture data" is actually a C string with the path to our texture in it // load it from an external image in our data path - char texname[SYS_MAX_PATH]; - snprintf(texname, sizeof(texname), FS_TEXTUREDIR "/%s.png", (const char*)rdp.loaded_texture[tile].addr); - load_texture(texname); -#else + if (is_file_texture((const char*)rdp.loaded_texture[tile].addr)) { + char texname[SYS_MAX_PATH]; + snprintf(texname, sizeof(texname), FS_TEXTUREDIR "/%s.png", (const char*)rdp.loaded_texture[tile].addr); + load_texture(texname); + return; + } +#endif // the texture data is actual texture data if (fmt == G_IM_FMT_RGBA) { if (siz == G_IM_SIZ_32b) { @@ -783,7 +857,6 @@ static void import_texture(int tile) { } else { sys_fatal("unsupported texture format: %u", fmt); } -#endif } static void gfx_normalize_vector(float v[3]) { @@ -1979,10 +2052,19 @@ void gfx_init(struct GfxWindowManagerAPI *wapi, struct GfxRenderingAPI *rapi, co gfx_lookup_or_create_shader_program(precomp_shaders[i]); } -#ifdef EXTERNAL_DATA +#if defined(EXTERNAL_DATA) && !defined(TARGET_PORT_CONSOLE) void gfx_precache_textures(void) { // preload all textures fs_walk(FS_TEXTUREDIR, preload_texture, NULL, true); + // print complete cache message once we are done + #ifdef _WIN32 + // clear with spaces and newline + fprintf(stdout, "\r%*s\rprecaching complete!\n", 80, ""); + #else + // use ANSI codes + fprintf(stdout, "\033[2K\rprecaching complete!\n"); + #endif + fflush(stdout); } #endif diff --git a/src/pc/pc_main.c b/src/pc/pc_main.c index 9d4d4e122..292a89a70 100644 --- a/src/pc/pc_main.c +++ b/src/pc/pc_main.c @@ -395,7 +395,7 @@ void main_func(void) { inited = true; -#ifdef EXTERNAL_DATA +#if defined(EXTERNAL_DATA) && !defined(TARGET_PORT_CONSOLE) // precache data if needed if (configPrecacheRes) { fprintf(stdout, "precaching data\n"); diff --git a/tools/mkzip.py b/tools/mkzip.py deleted file mode 100644 index 5481bc59e..000000000 --- a/tools/mkzip.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import os -import zipfile - -if len(sys.argv) < 3: - print('usage: mkzip ') - sys.exit(1) - -lst = [] -with open(sys.argv[1], 'r') as f: - for line in f: - line = line.strip() - if line == '' or line[0] == '#': - continue - tok = line.split() - lst.append((tok[0], tok[1])) - -with zipfile.ZipFile(sys.argv[2], 'w', allowZip64=False) as zipf: - for (fname, aname) in lst: - zipf.write(fname, arcname=aname)