diff --git a/ASADTool/ASADTool.cpp b/ASADTool/ASADTool.cpp new file mode 100644 index 0000000..d56b6c0 --- /dev/null +++ b/ASADTool/ASADTool.cpp @@ -0,0 +1,322 @@ +#include "WindowsUnicodeToolShim.h" +#include "PLBigEndian.h" +#include "MacFileInfo.h" +#include "CombinedTimestamp.h" +#include "CFileStream.h" +#include "PLCore.h" + +#include + +// https://tools.ietf.org/rfc/rfc1740 + +int ProcessFork(FILE *f, uint32_t length, const char *basePath, const char *suffix) +{ + const size_t kBufferSize = 4096; + + uint8_t buffer[kBufferSize]; + + std::string combinedPath = std::string(basePath) + suffix; + + FILE *outF = fopen_utf8(combinedPath.c_str(), "wb"); + if (!outF) + { + fprintf(stderr, "Failed to open output file '%s'", combinedPath.c_str()); + return -1; + } + + while (length > 0) + { + const size_t amountToCopy = std::min(length, kBufferSize); + + if (fread(buffer, 1, amountToCopy, f) != amountToCopy) + { + fprintf(stderr, "Failed to copy data"); + fclose(outF); + return -1; + } + + if (fwrite(buffer, 1, amountToCopy, outF) != amountToCopy) + { + fprintf(stderr, "Failed to copy data"); + fclose(outF); + return -1; + } + + length -= static_cast(amountToCopy); + } + + fclose(outF); + return 0; +} + +int ProcessFileDatesInfo(FILE *f, uint32_t length, PortabilityLayer::MacFileProperties &mfp, PortabilityLayer::CombinedTimestamp &ts) +{ + struct ASFileDates + { + BEInt32_t m_created; + BEInt32_t m_modified; + BEInt32_t m_backup; + BEInt32_t m_access; + }; + + ASFileDates fileDates; + if (length < sizeof(fileDates)) + { + fprintf(stderr, "File dates block was truncated"); + return -1; + } + + if (fread(&fileDates, 1, sizeof(fileDates), f) != sizeof(fileDates)) + { + fprintf(stderr, "Failed to read file dates"); + return -1; + } + + const int64_t asEpochToMacEpoch = -3029547600LL; + + // Mac epoch in Unix time: -2082844800 + // ASAD epoch in Unix time: 946702800 + + mfp.m_createdTimeMacEpoch = static_cast(fileDates.m_created) + asEpochToMacEpoch; + mfp.m_modifiedTimeMacEpoch = static_cast(fileDates.m_modified) + asEpochToMacEpoch; + ts.SetMacEpochTime(mfp.m_modifiedTimeMacEpoch); + + return 0; +} + +int ProcessFinderInfo(FILE *f, uint32_t length, PortabilityLayer::MacFileProperties &mfp) +{ + struct ASFinderInfo + { + uint8_t m_type[4]; + uint8_t m_creator[4]; + BEUInt16_t m_finderFlags; + BEPoint m_location; + BEUInt16_t m_folder; // ??? + }; + + struct ASExtendedFinderInfo + { + BEUInt16_t m_iconID; + uint8_t m_unused[6]; + uint8_t m_scriptCode; + uint8_t m_xFlags; + BEUInt16_t m_commentID; + BEUInt32_t m_putAwayDirectoryID; + }; + + ASFinderInfo finderInfo; + if (length < sizeof(finderInfo)) + { + fprintf(stderr, "Finder Info block was truncated"); + return -1; + } + + if (fread(&finderInfo, 1, sizeof(finderInfo), f) != sizeof(finderInfo)) + { + fprintf(stderr, "Failed to read Finder info"); + return -1; + } + + memcpy(mfp.m_fileCreator, finderInfo.m_creator, 4); + memcpy(mfp.m_fileType, finderInfo.m_type, 4); + mfp.m_finderFlags = finderInfo.m_finderFlags; + mfp.m_xPos = finderInfo.m_location.h; + mfp.m_yPos = finderInfo.m_location.v; + + return 0; +} + +int ProcessMacintoshFileInfo(FILE *f, uint32_t length, PortabilityLayer::MacFileProperties &mfp) +{ + struct ASMacInfo + { + uint8_t m_filler[3]; + uint8_t m_protected; + }; + + ASMacInfo macInfo; + if (length < sizeof(macInfo)) + { + fprintf(stderr, "File dates block was truncated"); + return -1; + } + + if (fread(&macInfo, 1, sizeof(macInfo), f) != sizeof(macInfo)) + { + fprintf(stderr, "Failed to read file dates"); + return -1; + } + + mfp.m_protected = macInfo.m_protected; + + return 0; +} + +int ProcessFile(FILE *f, const char *outPath, PortabilityLayer::CombinedTimestamp ts, bool isDouble) +{ + struct ASHeader + { + BEUInt32_t m_version; + uint8_t m_filler[16]; + BEUInt16_t m_numEntries; + }; + + struct ASEntry + { + BEUInt32_t m_entryID; + BEUInt32_t m_offset; + BEUInt32_t m_length; + }; + + ASHeader header; + if (fread(&header, 1, sizeof(header), f) != sizeof(header)) + { + fprintf(stderr, "Failed to read header"); + return -1; + } + + const uint32_t numEntries = header.m_numEntries; + + if (numEntries > 0xffff) + { + fprintf(stderr, "Too many entries"); + return -1; + } + + if (numEntries == 0) + return 0; + + std::vector entries; + entries.resize(static_cast(numEntries)); + + PortabilityLayer::MacFileProperties mfp; + + if (fread(&entries[0], 1, sizeof(ASEntry) * numEntries, f) != sizeof(ASEntry) * numEntries) + { + fprintf(stderr, "Failed to read entries"); + return -1; + } + + for (const ASEntry &asEntry : entries) + { + int fseekResult = fseek(f, asEntry.m_offset, SEEK_SET); + if (fseekResult != 0) + return fseekResult; + + int rc = 0; + switch (static_cast(asEntry.m_entryID)) + { + case 1: + if (asEntry.m_length > 0) + rc = ProcessFork(f, asEntry.m_length, outPath, ".gpd"); + break; + case 2: + if (asEntry.m_length > 0) + rc = ProcessFork(f, asEntry.m_length, outPath, ".gpr"); + break; + case 4: + if (asEntry.m_length > 0) + rc = ProcessFork(f, asEntry.m_length, outPath, ".gpc"); + break; + case 8: + rc = ProcessFileDatesInfo(f, asEntry.m_length, mfp, ts); + break; + case 9: + rc = ProcessFinderInfo(f, asEntry.m_length, mfp); + break; + case 10: + rc = ProcessMacintoshFileInfo(f, asEntry.m_length, mfp); + break; + case 3: // Real name + case 5: // B&W icon + case 6: // Color icon + case 11: // ProDOS file info + case 12: // MS-DOS file info + case 13: // AFP short name + case 14: // AFP file info + case 15: // AFP directory ID + break; + default: + fprintf(stderr, "Unknown entry type %i", static_cast(static_cast(asEntry.m_entryID))); + return -1; + } + + if (rc != 0) + return rc; + } + + PortabilityLayer::MacFilePropertiesSerialized mfps; + mfps.Serialize(mfp); + + std::string gpfPath = std::string(outPath) + ".gpf"; + + FILE *gpfFile = fopen_utf8(gpfPath.c_str(), "wb"); + if (!gpfFile) + { + fprintf(stderr, "Failed to open output gpf"); + return -1; + } + + PortabilityLayer::CFileStream gpfStream(gpfFile); + mfps.WriteAsPackage(gpfStream, ts); + + gpfStream.Close(); + + return 0; +} + +int toolMain(int argc, const char **argv) +{ + BEUInt32_t magic; + + if (argc != 4) + { + fprintf(stderr, "Usage: ASADTool "); + return -1; + } + + PortabilityLayer::CombinedTimestamp ts; + FILE *tsFile = fopen_utf8(argv[2], "rb"); + if (!tsFile) + { + fprintf(stderr, "Could not open timestamp file"); + return -1; + } + + if (fread(&ts, 1, sizeof(ts), tsFile) != sizeof(ts)) + { + fprintf(stderr, "Could not read timestamp file"); + return -1; + } + + fclose(tsFile); + + FILE *asadFile = fopen_utf8(argv[1], "rb"); + if (!asadFile) + { + fprintf(stderr, "Could not open input file"); + return -1; + } + + if (fread(&magic, 1, 4, asadFile) != 4) + { + fprintf(stderr, "Could not read file magic"); + return -1; + } + + int returnCode = 0; + if (magic == 0x00051607) + returnCode = ProcessFile(asadFile, argv[3], ts, true); + else if (magic == 0x00051600) + returnCode = ProcessFile(asadFile, argv[3], ts, false); + else + { + fprintf(stderr, "Unknown file type %x", static_cast(magic)); + return -1; + } + + fclose(asadFile); + + return returnCode; +} diff --git a/ASADTool/ASADTool.vcxproj b/ASADTool/ASADTool.vcxproj new file mode 100644 index 0000000..3933905 --- /dev/null +++ b/ASADTool/ASADTool.vcxproj @@ -0,0 +1,96 @@ + + + + + Debug + x64 + + + Release + x64 + + + + 15.0 + {DF692F94-3A11-40E1-8846-9815B4DBBDB0} + ASADTool + 10.0.17763.0 + + + + Application + true + v141 + MultiByte + + + Application + false + v141 + true + MultiByte + + + + + + + + + + + + + + + + + + + + + + + + + + + Level3 + MaxSpeed + true + true + true + true + + + Console + true + true + + + + + Level3 + Disabled + true + true + + + Console + + + + + {6ec62b0f-9353-40a4-a510-3788f1368b33} + + + {15009625-1120-405e-8bba-69a16cd6713d} + + + + + + + + + \ No newline at end of file diff --git a/ASADTool/ASADTool.vcxproj.filters b/ASADTool/ASADTool.vcxproj.filters new file mode 100644 index 0000000..616d2b6 --- /dev/null +++ b/ASADTool/ASADTool.vcxproj.filters @@ -0,0 +1,22 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hh;hpp;hxx;hm;inl;inc;ipp;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + + \ No newline at end of file diff --git a/Aerofoil.sln b/Aerofoil.sln index f40e54d..e3e1ac9 100644 --- a/Aerofoil.sln +++ b/Aerofoil.sln @@ -71,6 +71,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bin2h", "bin2h\bin2h.vcxpro EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "HouseTool", "HouseTool\HouseTool.vcxproj", "{B31BFF9D-2D14-4B1A-A625-8348CC3D8D67}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ASADTool", "ASADTool\ASADTool.vcxproj", "{DF692F94-3A11-40E1-8846-9815B4DBBDB0}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x64 = Debug|x64 @@ -195,6 +197,10 @@ Global {B31BFF9D-2D14-4B1A-A625-8348CC3D8D67}.Debug|x64.Build.0 = Debug|x64 {B31BFF9D-2D14-4B1A-A625-8348CC3D8D67}.Release|x64.ActiveCfg = Release|x64 {B31BFF9D-2D14-4B1A-A625-8348CC3D8D67}.Release|x64.Build.0 = Release|x64 + {DF692F94-3A11-40E1-8846-9815B4DBBDB0}.Debug|x64.ActiveCfg = Debug|x64 + {DF692F94-3A11-40E1-8846-9815B4DBBDB0}.Debug|x64.Build.0 = Debug|x64 + {DF692F94-3A11-40E1-8846-9815B4DBBDB0}.Release|x64.ActiveCfg = Release|x64 + {DF692F94-3A11-40E1-8846-9815B4DBBDB0}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/ConvertResources.bat b/ConvertResources.bat index f42e531..e8f4549 100644 --- a/ConvertResources.bat +++ b/ConvertResources.bat @@ -4,7 +4,7 @@ mkdir Packaged mkdir Packaged\Houses x64\Release\MiniRez.exe "GliderProData\Glider PRO.r" Packaged\ApplicationResources.gpr -x64\Release\gpr2gpa.exe "Packaged\ApplicationResources.gpr" "DefaultTimestamp.timestamp" "Packaged\ApplicationResources.gpa" "ApplicationResourcePatches\manifest.json" +x64\Release\gpr2gpa.exe "Packaged\ApplicationResources.gpr" "DefaultTimestamp.timestamp" "Packaged\ApplicationResources.gpa" -patch "ApplicationResourcePatches\manifest.json" x64\Release\FTagData.exe "DefaultTimestamp.timestamp" "Packaged\ApplicationResources.gpf" data ozm5 0 0 locked x64\Release\MergeGPF.exe "Packaged\ApplicationResources.gpf" @@ -54,14 +54,14 @@ x64\Release\gpr2gpa.exe "Packaged\Houses\CD Demo House.gpr" "DefaultTimestamp.ti x64\Release\gpr2gpa.exe "Packaged\Houses\Davis Station.gpr" "DefaultTimestamp.timestamp" "Packaged\Houses\Davis Station.gpa" x64\Release\gpr2gpa.exe "Packaged\Houses\Demo House.gpr" "DefaultTimestamp.timestamp" "Packaged\Houses\Demo House.gpa" x64\Release\gpr2gpa.exe "Packaged\Houses\Fun House.gpr" "DefaultTimestamp.timestamp" "Packaged\Houses\Fun House.gpa" -x64\Release\gpr2gpa.exe "Packaged\Houses\Grand Prix.gpr" "DefaultTimestamp.timestamp" "Packaged\Houses\Grand Prix.gpa" "HousePatches\GrandPrix.json" -x64\Release\gpr2gpa.exe "Packaged\Houses\ImagineHouse PRO II.gpr" "DefaultTimestamp.timestamp" "Packaged\Houses\ImagineHouse PRO II.gpa" "HousePatches\ImagineHousePROII.json" -x64\Release\gpr2gpa.exe "Packaged\Houses\In The Mirror.gpr" "DefaultTimestamp.timestamp" "Packaged\Houses\In The Mirror.gpa" "HousePatches\InTheMirror.json" +x64\Release\gpr2gpa.exe "Packaged\Houses\Grand Prix.gpr" "DefaultTimestamp.timestamp" "Packaged\Houses\Grand Prix.gpa" -patch "HousePatches\GrandPrix.json" +x64\Release\gpr2gpa.exe "Packaged\Houses\ImagineHouse PRO II.gpr" "DefaultTimestamp.timestamp" "Packaged\Houses\ImagineHouse PRO II.gpa" -patch "HousePatches\ImagineHousePROII.json" +x64\Release\gpr2gpa.exe "Packaged\Houses\In The Mirror.gpr" "DefaultTimestamp.timestamp" "Packaged\Houses\In The Mirror.gpa" -patch "HousePatches\InTheMirror.json" x64\Release\gpr2gpa.exe "Packaged\Houses\Land of Illusion.gpr" "DefaultTimestamp.timestamp" "Packaged\Houses\Land of Illusion.gpa" -x64\Release\gpr2gpa.exe "Packaged\Houses\Leviathan.gpr" "DefaultTimestamp.timestamp" "Packaged\Houses\Leviathan.gpa" "HousePatches\Leviathan.json" +x64\Release\gpr2gpa.exe "Packaged\Houses\Leviathan.gpr" "DefaultTimestamp.timestamp" "Packaged\Houses\Leviathan.gpa" -patch "HousePatches\Leviathan.json" x64\Release\gpr2gpa.exe "Packaged\Houses\Metropolis.gpr" "DefaultTimestamp.timestamp" "Packaged\Houses\Metropolis.gpa" x64\Release\gpr2gpa.exe "Packaged\Houses\Nemo's Market.gpr" "DefaultTimestamp.timestamp" "Packaged\Houses\Nemo's Market.gpa" -x64\Release\gpr2gpa.exe "Packaged\Houses\Rainbow's End.gpr" "DefaultTimestamp.timestamp" "Packaged\Houses\Rainbow's End.gpa" "HousePatches\RainbowsEnd.json" +x64\Release\gpr2gpa.exe "Packaged\Houses\Rainbow's End.gpr" "DefaultTimestamp.timestamp" "Packaged\Houses\Rainbow's End.gpa" -patch "HousePatches\RainbowsEnd.json" x64\Release\gpr2gpa.exe "Packaged\Houses\Slumberland.gpr" "DefaultTimestamp.timestamp" "Packaged\Houses\Slumberland.gpa" x64\Release\gpr2gpa.exe "Packaged\Houses\SpacePods.gpr" "DefaultTimestamp.timestamp" "Packaged\Houses\SpacePods.gpa" x64\Release\gpr2gpa.exe "Packaged\Houses\Teddy World.gpr" "DefaultTimestamp.timestamp" "Packaged\Houses\Teddy World.gpa" diff --git a/GpApp/HouseIO.cpp b/GpApp/HouseIO.cpp index 1714bc8..4d93dc0 100644 --- a/GpApp/HouseIO.cpp +++ b/GpApp/HouseIO.cpp @@ -2386,7 +2386,7 @@ static ExportHouseResult_t TryExportSound(GpVector &resData, const THan { BEUInt32_t m_samplePtr; BEUInt32_t m_length; - BEFixed32_t m_sampleRate; + BEUFixed32_t m_sampleRate; BEUInt32_t m_loopStart; BEUInt32_t m_loopEnd; uint8_t m_encoding; @@ -2784,10 +2784,10 @@ static ExportHouseResult_t TryExportPictFromSurface(GpVector &resData, BEUInt16_t m_headerOp; BEInt16_t m_v2Version; BEInt16_t m_reserved1; - BEFixed32_t m_top; - BEFixed32_t m_left; - BEFixed32_t m_bottom; - BEFixed32_t m_right; + BESFixed32_t m_top; + BESFixed32_t m_left; + BESFixed32_t m_bottom; + BESFixed32_t m_right; BEUInt32_t m_reserved2; }; diff --git a/MergeGPF/MergeGPF.cpp b/MergeGPF/MergeGPF.cpp index 783f8e1..87511eb 100644 --- a/MergeGPF/MergeGPF.cpp +++ b/MergeGPF/MergeGPF.cpp @@ -229,6 +229,7 @@ int toolMain(int argc, const char **argv) std::vector fileNameSizes; FILE *resF = fopen_utf8(resName.c_str(), "rb"); + if (resF) { PortabilityLayer::ZipEndOfCentralDirectoryRecord eocd; diff --git a/PortabilityLayer/PLBigEndian.h b/PortabilityLayer/PLBigEndian.h index bab2802..b30a547 100644 --- a/PortabilityLayer/PLBigEndian.h +++ b/PortabilityLayer/PLBigEndian.h @@ -244,8 +244,26 @@ typedef BEInteger BEInt32_t; typedef BEInteger BEUInt16_t; typedef BEInteger BEUInt32_t; -struct BEFixed32_t +struct BESFixed32_t { BEInt16_t m_intPart; BEUInt16_t m_fracPart; }; + +struct BEUFixed32_t +{ + BEUInt16_t m_intPart; + BEUInt16_t m_fracPart; +}; + +struct BESFixed16_t +{ + int8_t m_intPart; + uint8_t m_fracPart; +}; + +struct BEUFixed16_t +{ + uint8_t m_intPart; + uint8_t m_fracPart; +}; diff --git a/WindowsUnicodeToolShim/WindowsUnicodeToolShim.cpp b/WindowsUnicodeToolShim/WindowsUnicodeToolShim.cpp index a2c4bd7..5caae6a 100644 --- a/WindowsUnicodeToolShim/WindowsUnicodeToolShim.cpp +++ b/WindowsUnicodeToolShim/WindowsUnicodeToolShim.cpp @@ -1,9 +1,11 @@ #include #include "GpUnicode.h" +#include "WindowsUnicodeToolShim.h" #include #include +#include // This library provides front-ends and shims to make tools a bit more portable by handling all path strings as UTF-8, // and providing a "main" entry point that is also UTF-8. @@ -121,6 +123,58 @@ void ScanDirectoryForExtension(std::vector &outPaths, const char *p FindClose(h); } +struct DirectoryScanContext +{ + WIN32_FIND_DATAW m_findDataW; + HANDLE m_handle; + bool m_first; + + std::string m_utf8Name; + DirectoryScanEntry m_currentEntry; +}; + +static void ParseDirEntry(DirectoryScanContext &context) +{ + context.m_utf8Name = ConvertWStringToUTF8(context.m_findDataW.cFileName); + context.m_currentEntry.m_name = context.m_utf8Name.c_str(); +} + +DirectoryScanContext *opendir_utf8(const char *name) +{ + DirectoryScanContext *context = new DirectoryScanContext(); + + std::wstring dirFilter = std::wstring(L"\\\\?\\") + ConvertUTF8ToWString(name) + L"\\*"; + + context->m_handle = FindFirstFileW(dirFilter.c_str(), &context->m_findDataW); + if (context->m_handle == INVALID_HANDLE_VALUE) + { + delete context; + return nullptr; + } + + context->m_first = true; + return context; +} + +DirectoryScanEntry *readdir_utf8(DirectoryScanContext *dir) +{ + if (dir->m_first) + dir->m_first = false; + else + { + if (!FindNextFileW(dir->m_handle, &dir->m_findDataW)) + return nullptr; + } + + ParseDirEntry(*dir); + return &dir->m_currentEntry; +} + +void closedir_utf8(DirectoryScanContext *context) +{ + FindClose(context->m_handle); + delete context; +} int toolMain(int argc, const char **argv); diff --git a/WindowsUnicodeToolShim/WindowsUnicodeToolShim.h b/WindowsUnicodeToolShim/WindowsUnicodeToolShim.h index c4f3469..8d94ab5 100644 --- a/WindowsUnicodeToolShim/WindowsUnicodeToolShim.h +++ b/WindowsUnicodeToolShim/WindowsUnicodeToolShim.h @@ -7,3 +7,13 @@ int fputs_utf8(const char *str, FILE *f); int mkdir_utf8(const char *path); void TerminateDirectoryPath(std::string &path); void ScanDirectoryForExtension(std::vector& outPaths, const char *path, const char *ending, bool recursive); + +struct DirectoryScanContext; +struct DirectoryScanEntry +{ + const char *m_name; +}; + +DirectoryScanContext *opendir_utf8(const char *name); +DirectoryScanEntry *readdir_utf8(DirectoryScanContext *dir); +void closedir_utf8(DirectoryScanContext *context); diff --git a/bulkimport.py b/bulkimport.py new file mode 100644 index 0000000..51d4160 --- /dev/null +++ b/bulkimport.py @@ -0,0 +1,417 @@ +import sys +import os +import subprocess +import zipfile +import shutil +import io +import json + +debug_preserve_osx_dir = False +debug_preserve_temps = True +debug_preserve_resources = True +debug_preserve_qt = False + + +def invoke_command(process_path, args, output_lines=None): + print("Running " + str(process_path) + " with " + str(args)) + args_concatenated = [process_path + ".exe"] + args + if output_lines != None: + completed_process = subprocess.run(args_concatenated, capture_output=True) + else: + completed_process = subprocess.run(args_concatenated) + + if completed_process.returncode != 0: + print("Process crashed/failed with return code " + str(completed_process.returncode)) + return False + + if output_lines != None: + output_lines.clear() + output_lines.extend(completed_process.stdout.decode("utf-8", "ignore").splitlines(False)) + + return True + +def recursive_scan_dir(out_paths, dir_path): + dir = os.scandir(dir_path) + for entry in dir: + if entry.is_dir(): + recursive_scan_dir(out_paths, entry.path) + if entry.is_file(): + out_paths.append(entry.path) + +def decompress_zip(ftagdata_path, source_path, ts_path, decompress_path): + with zipfile.ZipFile(source_path, "r") as zfile: + zfile.extractall(decompress_path) + + file_names = [] + recursive_scan_dir(file_names, decompress_path) + + for path in file_names: + if not invoke_command(ftagdata_path, [ts_path, path + ".gpf", "DATA", "DATA", "0", "0"]): + return False + os.replace(path, path + ".gpd") + + return True + +def fixup_macos_dir(ftagdata_path, asadtool_path, ts_path, dir_path, osx_path): + contents = [] + recursive_scan_dir(contents, osx_path) + + print("recursive_scan_dir results: " + str(contents)) + for content_path in contents: + osx_rel_path = os.path.relpath(content_path, osx_path) + osx_rel_dir, osx_rel_file = os.path.split(osx_rel_path) + + if osx_rel_file.startswith("._") and osx_rel_file.endswith(".gpd"): + out_path = os.path.join(dir_path, osx_rel_dir, osx_rel_file[2:-4]) + if not invoke_command(asadtool_path, [content_path, ts_path, out_path]): + return False + + return True + +def recursive_fixup_macosx_dir(ftagdata_path, asadtool_path, ts_path, dir_path): + osx_path = os.path.join(dir_path, "__MACOSX") + if os.path.isdir(osx_path): + if not fixup_macos_dir(ftagdata_path, asadtool_path, ts_path, dir_path, osx_path): + print("fixup_macos_dir failed?") + return False + if not debug_preserve_osx_dir: + shutil.rmtree(osx_path) + + dir = os.scandir(dir_path) + for entry in dir: + if entry.is_dir(): + if not recursive_fixup_macosx_dir(ftagdata_path, asadtool_path, ts_path, entry.path): + return False + + return True + +def convert_movies(tools_dir, dir_path): + contents = [] + recursive_scan_dir(contents, dir_path) + for content_path in contents: + print("convert_movies content path: " + content_path) + if content_path.endswith(".mov.gpf"): + if not os.path.isfile(content_path[:-4] + ".gpd"): + # Res-only movie, probably only contains external references, a.k.a. unusable + os.remove(content_path) + if os.path.isfile(content_path[:-4] + ".gpr"): + os.remove(content_path[:-4] + ".gpr") + else: + content_dir = os.path.dirname(content_path) + mov_path = content_path[:-4] + res_path = mov_path + ".gpr" + data_path = mov_path + ".gpd" + if os.path.isfile(res_path): + if not invoke_command(os.path.join(tools_dir, "flattenmov"), [data_path, res_path, mov_path]): + return False + + if not debug_preserve_qt: + os.remove(res_path) + os.remove(data_path) + else: + if os.path.isfile(mov_path): + os.remove(mov_path) + os.rename(data_path, mov_path) + + probe_lines = [] + if not invoke_command(os.path.join(tools_dir, "ffprobe"), ["-show_streams", mov_path], probe_lines): + return False + + v_index = None + v_fps_num = None + v_fps_denom = None + + a_index = None + a_nbframes = None + a_sample_rate = None + + current_fps = None + current_index = None + current_type = None + current_nbframes = None + current_sample_rate = None + is_stream = False + for l in probe_lines: + if is_stream: + if l == "[/STREAM]": + print("Closing stream: " + str(current_type) + " " + str(current_index) + " " + str(current_fps) + " " + str(current_nbframes) + " " + str(current_sample_rate)) + if current_type == "video" and current_index != None and current_fps != None: + fps_list = current_fps.split("/") + v_index = current_index + v_fps_num = fps_list[0] + v_fps_denom = fps_list[1] + if current_type == "audio" and current_index != None and current_nbframes != None and current_sample_rate != None: + a_index = current_index + a_nbframes = current_nbframes + a_sample_rate = current_sample_rate + + current_fps = None + current_index = None + current_type = None + current_nbframes = None + current_sample_rate = None + is_stream = False + elif l.startswith("codec_type="): + current_type = l[11:] + elif l.startswith("index="): + current_index = l[6:] + elif l.startswith("r_frame_rate="): + current_fps = l[13:] + elif l.startswith("nb_frames="): + current_nbframes = l[10:] + elif l.startswith("sample_rate="): + current_sample_rate = l[12:] + elif l == "[STREAM]": + current_fps_num = None + current_fps_denom = None + current_index = None + current_type = None + is_stream = True + + wav_path = None + if a_index != None: + sample_rate_int = int(a_sample_rate) + target_sample_rate = "22254" + if sample_rate_int == 11025 or sample_rate_int == 44100: + target_sample_rate = "22050" + elif sample_rate_int < 22000 or sample_rate_int > 23000: + target_sample_rate = a_sample_rate + + wav_path = os.path.join(content_dir, "0.wav") + if not invoke_command(os.path.join(tools_dir, "ffmpeg"), ["-y", "-i", mov_path, "-ac", "1", "-ar", target_sample_rate, "-c:a", "pcm_u8", wav_path]): + return False + + if v_index != None: + if not invoke_command(os.path.join(tools_dir, "ffmpeg"), ["-y", "-i", mov_path, os.path.join(content_dir, "%d.bmp")]): + return False + + if a_index != None or v_index != None: + with zipfile.ZipFile(mov_path + ".gpa", "w") as vid_archive: + metaf = io.StringIO() + + if v_index != None: + metaf.write("{\n") + metaf.write("\t\"frameRateNumerator\" : " + v_fps_num + ",\n") + metaf.write("\t\"frameRateDenominator\" : " + v_fps_denom + "\n") + metaf.write("}\n") + else: + metaf.write("{\n") + metaf.write("\t\"frameRateNumerator\" : " + a_nbframes + ",\n") + metaf.write("\t\"frameRateDenominator\" : " + a_sample_rate + "\n") + metaf.write("}\n") + + vid_archive.writestr("muvi/0.json", metaf.getvalue(), compress_type=zipfile.ZIP_DEFLATED, compresslevel=9) + + if v_index != None: + frame_num = 1 + bmp_name = str(frame_num) + ".bmp" + bmp_path = os.path.join(content_dir, bmp_name) + while os.path.isfile(bmp_path): + vid_archive.write(bmp_path, arcname=("PICT/" + bmp_name), compress_type=zipfile.ZIP_DEFLATED, compresslevel=9) + os.remove(bmp_path) + frame_num = frame_num + 1 + bmp_name = str(frame_num) + ".bmp" + bmp_path = os.path.join(content_dir, bmp_name) + + if a_index != None: + vid_archive.write(wav_path, arcname=("snd$20/0.wav"), compress_type=zipfile.ZIP_DEFLATED, compresslevel=9) + os.remove(wav_path) + + if not debug_preserve_qt: + os.remove(mov_path) + + return True + +def reprocess_children(source_paths, dir_path): + reprocess_extensions = [ "sea", "bin", "hqx", "zip", "cpt", "sit" ] + contents = [] + recursive_scan_dir(contents, dir_path) + for ext in reprocess_extensions: + full_ext = "." + ext + ".gpf" + for content_path in contents: + if content_path.endswith(full_ext): + truncated_path = content_path[:-4] + data_path = truncated_path + ".gpd" + if os.path.isfile(data_path): + os.rename(data_path, truncated_path) + source_paths.append(truncated_path) + print("Requeueing subpath " + truncated_path) + + return True + +def convert_resources(tools_dir, ts_path, qt_convert_dir, dir_path): + contents = [] + recursive_scan_dir(contents, dir_path) + for content_path in contents: + if content_path.endswith(".gpr"): + if not invoke_command(os.path.join(tools_dir, "gpr2gpa"), [content_path, ts_path, content_path[:-4] + ".gpa", "-dumpqt", qt_convert_dir]): + return False + + qt_convert_contents = [] + recursive_scan_dir(qt_convert_contents, qt_convert_dir) + + converted_pict_ids = [] + + # Convert inline QuickTime PICT resources + for convert_content_path in qt_convert_contents: + if convert_content_path.endswith(".mov"): + if not invoke_command(os.path.join(tools_dir, "ffmpeg"), ["-y", "-i", convert_content_path, convert_content_path[:-4] + ".bmp"]): + return False + os.remove(convert_content_path) + + converted_pict_ids.append(os.path.basename(convert_content_path[:-4])) + + if len(converted_pict_ids) > 0: + print("Reimporting converted QuickTime PICTs") + qt_convert_json_path = os.path.join(dir_path, "qt_convert.json") + + convert_dict = { } + convert_dict["delete"] = [] + convert_dict["add"] = { } + + for pict_id in converted_pict_ids: + convert_dict["add"]["PICT/" + pict_id + ".bmp"] = os.path.join(qt_convert_dir, pict_id + ".bmp") + + with open(qt_convert_json_path, "w") as f: + json.dump(convert_dict, f) + + if not invoke_command(os.path.join(tools_dir, "gpr2gpa"), [content_path, ts_path, content_path[:-4] + ".gpa", "-patch", qt_convert_json_path]): + return False + + for pict_id in converted_pict_ids: + os.remove(os.path.join(qt_convert_dir, pict_id + ".bmp")) + os.remove(qt_convert_json_path) + + if not debug_preserve_resources: + os.remove(content_path) + + return True + +def scoop_files(tools_dir, output_dir, dir_path): + mergegpf_path = os.path.join(tools_dir, "MergeGPF") + contents = [] + recursive_scan_dir(contents, dir_path) + scooped_files = [] + for content_path in contents: + if content_path.endswith(".gpf"): + is_house = False + with zipfile.ZipFile(content_path, "r") as zfile: + meta_contents = None + with zfile.open("!!meta", "r") as metafile: + meta_contents = metafile.read() + if meta_contents[0] == 103 and meta_contents[1] == 108 and meta_contents[2] == 105 and meta_contents[3] == 72: + is_house = True + + if is_house: + if not invoke_command(mergegpf_path, [content_path]): + return False + scooped_files.append(content_path) + + mov_path = content_path[:-4] + ".mov.gpf" + if os.path.isfile(mov_path): + if not invoke_command(mergegpf_path, [mov_path]): + return False + scooped_files.append(mov_path) + + for scoop_path in scooped_files: + os.replace(scoop_path, os.path.join(output_dir, os.path.basename(scoop_path))) + + return True + +class ImportContext: + def __init__(self): + pass + + def run(self): + os.makedirs(self.qt_convert_dir, exist_ok=True) + os.makedirs(self.output_dir, exist_ok=True) + + invoke_command(self.make_timestamp_path, [self.ts_path]) + + print("Looking for input files in " + self.source_dir) + + source_paths = [] + recursive_scan_dir(source_paths, self.source_dir) + + pending_result_directories = [] + result_dir_index = 0 + + while len(source_paths) > 0: + source_path = source_paths[0] + source_paths = source_paths[1:] + + unpack_dir = os.path.join(self.output_dir, str(len(pending_result_directories))) + try: + os.mkdir(unpack_dir) + except FileExistsError as error: + pass + + print("Attempting to unpack " + source_path) + decompressed_ok = False + should_decompress = True + if source_path.endswith(".zip"): + decompressed_ok = decompress_zip(self.ftagdata_path, source_path, self.ts_path, unpack_dir) + elif source_path.endswith(".sit") or source_path.endswith(".cpt") or source_path.endswith(".sea"): + decompressed_ok = invoke_command(os.path.join(self.tools_dir, "unpacktool"), [source_path, self.ts_path, unpack_dir, "-paranoid"]) + elif source_path.endswith(".bin"): + decompressed_ok = invoke_command(os.path.join(self.tools_dir, "bin2gp"), [source_path, self.ts_path, os.path.join(unpack_dir, os.path.basename(source_path[:-4]))]) + elif source_path.endswith(".hqx"): + decompressed_ok = invoke_command(os.path.join(self.tools_dir, "hqx2gp"), [source_path, self.ts_path, os.path.join(unpack_dir, os.path.basename(source_path[:-4]))]) + else: + should_decompress = False + + if should_decompress and not decompressed_ok: + return + + if decompressed_ok: + pending_result_directories.append(unpack_dir) + + while result_dir_index < len(pending_result_directories): + if not self.process_dir(pending_result_directories, result_dir_index, source_paths): + return + + result_dir_index = result_dir_index + 1 + + # Clear temporaries + if not debug_preserve_temps: + for dir_path in pending_result_directories: + shutil.rmtree(dir_path) + + + def process_dir(self, all_dirs, dir_index, source_paths): + root = all_dirs[dir_index] + print("Processing directory " + root) + + if not recursive_fixup_macosx_dir(self.ftagdata_path, os.path.join(self.tools_dir, "ASADTool"), self.ts_path, root): + return False + + if not convert_movies(self.tools_dir, root): + return False + if not convert_resources(self.tools_dir, self.ts_path, self.qt_convert_dir, root): + return False + if not reprocess_children(source_paths, root): + return False + if not scoop_files(self.tools_dir, self.output_dir, root): + return False + + return True + + +def main(): + import_context = ImportContext() + + #script_dir = sys.argv[0] + #source_dir = sys.argv[1] + import_context.source_dir = "C:\\Users\\Eric\\Downloads\\gliderfiles\\archives" + #output_dir = sys.argv[2] + import_context.output_dir = "C:\\Users\\Eric\\Downloads\\gliderfiles\\converted" + + import_context.qt_convert_dir = os.path.join(import_context.output_dir, "qtconvert") + import_context.tools_dir = "D:\\src\\GlidePort\\x64\\Release" + import_context.make_timestamp_path = os.path.join(import_context.tools_dir, "MakeTimestamp") + import_context.ts_path = os.path.join(import_context.output_dir, "Timestamp.ts") + import_context.ftagdata_path = os.path.join(import_context.tools_dir, "FTagData") + + import_context.run() + +main() diff --git a/flattenmov/flattenmov.cpp b/flattenmov/flattenmov.cpp index 0c7b24b..b1787f4 100644 --- a/flattenmov/flattenmov.cpp +++ b/flattenmov/flattenmov.cpp @@ -2,6 +2,8 @@ #include "MacFileMem.h" #include "CFileStream.h" #include "MemReaderStream.h" +#include "PLDrivers.h" +#include "PLBigEndian.h" #include "ResourceCompiledTypeList.h" #include "ResourceFile.h" #include "ScopedPtr.h" @@ -50,63 +52,106 @@ int main(int argc, const char **argv) mfi.m_resourceForkSize = resSize; mfi.m_commentSize = 0; + GpDriverCollection *drivers = PLDrivers::GetDriverCollection(); + drivers->SetDriver(GpAllocator_C::GetInstance()); + PortabilityLayer::ScopedPtr memFile = PortabilityLayer::MacFileMem::Create(GpAllocator_C::GetInstance(), dataFork, resFork, nullptr, mfi); delete[] dataFork; delete[] resFork; const uint8_t *dataBytes = memFile->DataFork(); - if (dataBytes[0] == 0 && dataBytes[1] == 0 && dataBytes[2] == 0 && dataBytes[3] == 0) + + size_t terminalAtomPos = 0; + const size_t dataForkSize = memFile->FileInfo().m_dataForkSize; + + bool ignoreAndCopy = false; + if (dataBytes[0] == 'F' && dataBytes[1] == 'O' && dataBytes[2] == 'R' && dataBytes[3] == 'M') { - uint32_t mdatSize = memFile->FileInfo().m_dataForkSize; - uint8_t mdatSizeEncoded[4]; - mdatSizeEncoded[0] = ((mdatSize >> 24) & 0xff); - mdatSizeEncoded[1] = ((mdatSize >> 16) & 0xff); - mdatSizeEncoded[2] = ((mdatSize >> 8) & 0xff); - mdatSizeEncoded[3] = ((mdatSize >> 0) & 0xff); - - PortabilityLayer::ResourceFile *rf = PortabilityLayer::ResourceFile::Create(); - - PortabilityLayer::MemReaderStream resStream(memFile->ResourceFork(), memFile->FileInfo().m_resourceForkSize); - rf->Load(&resStream); - - const PortabilityLayer::ResourceCompiledTypeList *typeList = rf->GetResourceTypeList(PortabilityLayer::ResTypeID('moov')); - const uint8_t *moovResBytes = nullptr; - uint32_t moovResSize = 0; - for (size_t refIndex = 0; refIndex < typeList->m_numRefs; refIndex++) - { - const PortabilityLayer::ResourceCompiledRef &ref = typeList->m_firstRef[refIndex]; - moovResSize = ref.GetSize(); - moovResBytes = ref.m_resData; - break; - } - - FILE *outF = fopen(argv[3], "wb"); - if (!outF) - { - fprintf(stderr, "Could not open output file '%s'", argv[3]); - return -1; - } - - fwrite(mdatSizeEncoded, 1, 4, outF); - fwrite(dataBytes + 4, 1, mdatSize - 4, outF); - fwrite(moovResBytes, 1, moovResSize, outF); - - fclose(outF); - rf->Destroy(); + fprintf(stderr, "File appears to actually be an AIFF file\n"); + ignoreAndCopy = true; } + + const uint8_t *moovResBytes = nullptr; + uint32_t moovResSize = 0; + PortabilityLayer::ResourceFile *rf = nullptr; + + if (ignoreAndCopy) + terminalAtomPos = dataForkSize; else { - FILE *outF = fopen(argv[3], "wb"); - if (!outF) + while (terminalAtomPos < dataForkSize) { - fprintf(stderr, "Could not open output file '%s'", argv[3]); - return -1; + size_t szAvailable = dataForkSize - terminalAtomPos; + + if (szAvailable < 4) + { + fprintf(stderr, "Error looking for terminal atom"); + return -1; + } + + BEUInt32_t atomSize; + memcpy(&atomSize, dataBytes + terminalAtomPos, 4); + + if (atomSize == 0) + break; + + if (szAvailable < atomSize) + { + fprintf(stderr, "Error looking for terminal atom"); + return -1; + } + + terminalAtomPos += atomSize; } - fwrite(dataBytes, 1, memFile->FileInfo().m_dataForkSize, outF); - fclose(outF); + rf = PortabilityLayer::ResourceFile::Create(); + + if (rf) + { + + PortabilityLayer::MemReaderStream resStream(memFile->ResourceFork(), memFile->FileInfo().m_resourceForkSize); + rf->Load(&resStream); + + const PortabilityLayer::ResourceCompiledTypeList *typeList = rf->GetResourceTypeList(PortabilityLayer::ResTypeID('moov')); + + if (typeList != nullptr) + { + for (size_t refIndex = 0; refIndex < typeList->m_numRefs; refIndex++) + { + const PortabilityLayer::ResourceCompiledRef &ref = typeList->m_firstRef[refIndex]; + moovResSize = ref.GetSize(); + moovResBytes = ref.m_resData; + break; + } + } + } } + FILE *outF = fopen(argv[3], "wb"); + if (!outF) + { + fprintf(stderr, "Could not open output file '%s'", argv[3]); + return -1; + } + + if (terminalAtomPos > 0) + fwrite(dataBytes, 1, terminalAtomPos, outF); + + if (terminalAtomPos < dataForkSize) + { + BEUInt32_t atomSize(static_cast(dataForkSize - terminalAtomPos)); + fwrite(&atomSize, 1, 4, outF); + fwrite(dataBytes + terminalAtomPos + 4, 1, dataForkSize - terminalAtomPos - 4, outF); + } + + if (moovResBytes) + fwrite(moovResBytes, 1, moovResSize, outF); + + fclose(outF); + + if (rf) + rf->Destroy(); + return 0; } diff --git a/gpr2gpa/gpr2gpa.cpp b/gpr2gpa/gpr2gpa.cpp index c4c4830..93c0df4 100644 --- a/gpr2gpa/gpr2gpa.cpp +++ b/gpr2gpa/gpr2gpa.cpp @@ -29,9 +29,11 @@ #include "macedec.h" #include +#include +#include #include #include -#include +#include "GpWindows.h" enum AudioCompressionCodecID { @@ -498,6 +500,8 @@ bool ExportBMP(size_t width, size_t height, size_t pitchInElements, const Portab class BMPDumperContext : public PortabilityLayer::QDPictEmitContext { public: + explicit BMPDumperContext(const char *dumpqtDir, int resID); + bool SpecifyFrame(const Rect &rect) override; Rect ConstrainRegion(const Rect &rect) const override; void Start(PortabilityLayer::QDPictBlitSourceType sourceType, const PortabilityLayer::QDPictEmitScanlineParameters ¶ms) override; @@ -520,9 +524,19 @@ private: PortabilityLayer::QDPictEmitScanlineParameters m_blitParams; PortabilityLayer::QDPictBlitSourceType m_blitSourceType; + const char *m_dumpqtDir; + int m_resID; + std::vector m_tempBuffers; }; + +BMPDumperContext::BMPDumperContext(const char *dumpqtDir, int resID) + : m_dumpqtDir(dumpqtDir) + , m_resID(resID) +{ +} + bool BMPDumperContext::SpecifyFrame(const Rect &rect) { if (!rect.IsValid()) @@ -648,11 +662,705 @@ void BMPDumperContext::BlitScanlineAndAdvance(const void *scanlineData) } } + +struct ImageDescriptionData +{ + char m_codecType[4]; + uint8_t m_reserved[8]; + BEUInt16_t m_cdataVersion; + BEUInt16_t m_cdataRevision; + char m_vendor[4]; + BEUInt32_t m_temporalQuality; + BEUInt32_t m_spatialQuality; + BEUInt16_t m_width; + BEUInt16_t m_height; + BEUFixed32_t m_hRes; + BEUFixed32_t m_vRes; + BEUInt32_t m_dataSize; + BEUInt16_t m_frameCount; + uint8_t m_nameLength; + uint8_t m_name[31]; + BEUInt16_t m_depth; + BEUInt16_t m_clutID; +}; + +struct ImageDescription +{ + BEUInt32_t m_idSize; + ImageDescriptionData m_data; +}; + +class MovDumpScope +{ +public: + MovDumpScope(GpIOStream &stream, const char *atom); + ~MovDumpScope(); + +private: + GpIOStream &m_stream; + GpUFilePos_t m_startPos; +}; + +MovDumpScope::MovDumpScope(GpIOStream &stream, const char *atom) + : m_stream(stream) + , m_startPos(stream.Tell()) +{ + struct AtomData + { + BEUInt32_t m_size; + char m_type[4]; + }; + + AtomData atomData; + memcpy(atomData.m_type, atom, 4); + atomData.m_size = 0; + + stream.Write(&atomData, 8); +} + +MovDumpScope::~MovDumpScope() +{ + GpUFilePos_t returnPos = m_stream.Tell(); + m_stream.SeekStart(m_startPos); + BEUInt32_t size = BEUInt32_t(static_cast(returnPos - m_startPos)); + + m_stream.Write(&size, 4); + m_stream.SeekStart(returnPos); +} + + +static bool DumpMOV(GpIOStream &stream, const ImageDescription &imageDesc, const std::vector &imageData) +{ + if (imageData.size() == 0) + return true; + + struct HDLRData + { + uint8_t m_version; + uint8_t m_flags[3]; + BEUInt32_t m_componentType; + BEUInt32_t m_componentSubtype; + BEUInt32_t m_componentManufacturer; + BEUInt32_t m_componentFlags; + BEUInt32_t m_componentFlagsMask; + // Var: Name + }; + + const uint32_t duration = 40; + + { + MovDumpScope ftypScope(stream, "ftyp"); + + struct FTYPData + { + char m_majorBrand[4]; + BEUInt32_t m_minorVersion; + char m_compatibleBrands[4]; + }; + + FTYPData data; + memcpy(data.m_majorBrand, "qt ", 4); + data.m_minorVersion = 0x200; + memcpy(data.m_compatibleBrands, "qt ", 4); + + stream.Write(&data, sizeof(data)); + } + + { + MovDumpScope wideScope(stream, "wide"); + } + + GpUFilePos_t imageDataStart = 0; + { + MovDumpScope mdatScope(stream, "mdat"); + + imageDataStart = stream.Tell(); + stream.Write(&imageData[0], imageData.size()); + } + + { + MovDumpScope moovScope(stream, "moov"); + + { + MovDumpScope mvhdScope(stream, "mvhd"); + + struct MVHDData + { + uint8_t m_version; + uint8_t m_flags[3]; + BEUInt32_t m_creationTime; + BEUInt32_t m_modificationTime; + BEUInt32_t m_timeScale; + BEUInt32_t m_duration; + BEUFixed32_t m_preferredRate; + BEUFixed16_t m_preferredVolume; + uint8_t m_reserved[10]; + BESFixed32_t m_matrix[9]; + BEUInt32_t m_previewTime; + BEUInt32_t m_previewDuration; + BEUInt32_t m_posterTime; + BEUInt32_t m_selectionTime; + BEUInt32_t m_selectionDuration; + BEUInt32_t m_currentTime; + BEUInt32_t m_nextTrackID; + }; + + MVHDData mvhdData; + memset(&mvhdData, 0, sizeof(mvhdData)); + mvhdData.m_timeScale = 1000; + mvhdData.m_duration = duration; + mvhdData.m_preferredRate.m_intPart = 1; + mvhdData.m_preferredVolume.m_intPart = 1; + mvhdData.m_matrix[0].m_intPart = 1; + mvhdData.m_matrix[5].m_intPart = 5; + mvhdData.m_matrix[8].m_intPart = 0x4000; + mvhdData.m_nextTrackID = 2; + + stream.Write(&mvhdData, sizeof(mvhdData)); + } + + { + MovDumpScope trakScope(stream, "trak"); + + { + MovDumpScope tkhdScope(stream, "tkhd"); + + struct TKHDData + { + uint8_t m_version; + uint8_t m_flags[3]; + BEUInt32_t m_creationTime; + BEUInt32_t m_modificationTime; + BEUInt32_t m_trackID; + uint8_t m_reserved[4]; + BEUInt32_t m_duration; + uint8_t m_reserved2[8]; + BEUInt16_t m_layer; + BEUInt16_t m_alternateGroup; + BEUInt16_t m_volume; + uint8_t m_reserved3[2]; + BEUFixed32_t m_matrix[9]; + BEUFixed32_t m_trackWidth; + BEUFixed32_t m_trackHeight; + }; + + TKHDData tkhdData; + memset(&tkhdData, 0, sizeof(tkhdData)); + + tkhdData.m_flags[2] = 0x3; // Used in movie + enabled + tkhdData.m_trackID = 1; + tkhdData.m_duration = duration; + tkhdData.m_matrix[0].m_intPart = 1; + tkhdData.m_matrix[5].m_intPart = 5; + tkhdData.m_matrix[8].m_intPart = 0x4000; + tkhdData.m_trackWidth.m_intPart = imageDesc.m_data.m_width; + tkhdData.m_trackHeight.m_intPart = imageDesc.m_data.m_height; + + stream.Write(&tkhdData, sizeof(tkhdData)); + } + + { + MovDumpScope edtsScope(stream, "edts"); + + { + MovDumpScope elstScope(stream, "elst"); + + struct ELSTEntry + { + BEUInt32_t m_duration; + BEUInt32_t m_mediaTime; + BEUFixed32_t m_mediaRate; + }; + + struct ELSTData + { + uint8_t m_version; + uint8_t m_flags[3]; + BEUInt32_t m_numEntries; + + ELSTEntry m_entry; + }; + + ELSTData elstData; + memset(&elstData, 0, sizeof(elstData)); + + elstData.m_numEntries = 1; + elstData.m_entry.m_duration = duration; + elstData.m_entry.m_mediaRate.m_intPart = 1; + + stream.Write(&elstData, sizeof(elstData)); + } + } + + { + MovDumpScope mdiaScope(stream, "mdia"); + + { + MovDumpScope mdhdScope(stream, "mdhd"); + + struct MDHDData + { + uint8_t m_version; + uint8_t m_flags[3]; + BEUInt32_t m_creationTime; + BEUInt32_t m_modificationTime; + BEUFixed32_t m_timeScale; + BEUInt32_t m_duration; + BEUInt16_t m_language; + BEUInt16_t m_quality; + }; + + MDHDData mdhdData; + memset(&mdhdData, 0, sizeof(mdhdData)); + + mdhdData.m_timeScale.m_fracPart = 12800; + mdhdData.m_duration = 0x200; + mdhdData.m_language = 0x7fff; + + stream.Write(&mdhdData, sizeof(mdhdData)); + } + + { + MovDumpScope hdlrScope(stream, "hdlr"); + + HDLRData hdlrData; + memset(&hdlrData, 0, sizeof(hdlrData)); + + hdlrData.m_componentType = 0x6d686c72; // mhlr + hdlrData.m_componentSubtype = 0x76696465; // vide + + stream.Write(&hdlrData, sizeof(hdlrData)); + + const char *handlerName = "VideoHandler"; + uint8_t handlerNameLen = strlen(handlerName); + + stream.Write(&handlerNameLen, 1); + stream.Write(handlerName, handlerNameLen); + } + + { + MovDumpScope minfScope(stream, "minf"); + + { + MovDumpScope vmhdScope(stream, "vmhd"); + + struct VMHDData + { + uint8_t m_version; + uint8_t m_flags[3]; + BEUInt16_t m_graphicsMode; + BEUInt16_t m_opColor[3]; + }; + + VMHDData vmhdData; + memset(&vmhdData, 0, sizeof(vmhdData)); + + vmhdData.m_flags[2] = 1; // Compatibility flag + + stream.Write(&vmhdData, sizeof(vmhdData)); + } + + { + MovDumpScope hdlrScope(stream, "hdlr"); + + HDLRData hdlrData; + memset(&hdlrData, 0, sizeof(hdlrData)); + + hdlrData.m_componentType = 0x64686c72; // dhlr + hdlrData.m_componentSubtype = 0x75726c20; // url + + stream.Write(&hdlrData, sizeof(hdlrData)); + + const char *handlerName = "VideoHandler"; + uint8_t handlerNameLen = strlen(handlerName); + + stream.Write(&handlerNameLen, 1); + stream.Write(handlerName, handlerNameLen); + } + + { + MovDumpScope dinfScope(stream, "dinf"); + + { + MovDumpScope drefScope(stream, "dref"); + + struct DREFData + { + uint8_t m_version; + uint8_t m_flags[3]; + BEUInt32_t m_numEntries; + // DREFEntry + }; + + struct DREFEntry + { + BEUInt32_t m_size; + BEUInt32_t m_type; + uint8_t m_version; + uint8_t m_flags[3]; + // Data + }; + + DREFData drefData; + memset(&drefData, 0, sizeof(drefData)); + + drefData.m_numEntries = 1; + + stream.Write(&drefData, sizeof(drefData)); + + DREFEntry drefEntry; + memset(&drefEntry, 0, sizeof(drefEntry)); + + drefEntry.m_size = sizeof(DREFEntry); + drefEntry.m_type = 0x75726c20; // url + drefEntry.m_flags[2] = 1; // Self-contained + + stream.Write(&drefEntry, sizeof(drefEntry)); + } + } + + { + MovDumpScope stblScope(stream, "stbl"); + + { + MovDumpScope stsdScope(stream, "stsd"); + + struct STSDData + { + uint8_t m_version; + uint8_t m_flags[3]; + BEUInt32_t m_numEntries; + }; + + STSDData stsdData; + memset(&stsdData, 0, sizeof(stsdData)); + stsdData.m_numEntries = 1; + + stream.Write(&stsdData, sizeof(stsdData)); + + struct STSDEntry + { + BEUInt32_t m_size; + uint8_t m_dataFormat[4]; + uint8_t m_reserved[6]; + BEUInt16_t m_dataRefIndex; + }; + + struct FieldAtom + { + BEUInt32_t m_atomSize; + char m_atomID[4]; + uint8_t m_fieldCount; + uint8_t m_fieldOrdering; + }; + + struct VideoMediaSampleDescription + { + BEUInt16_t m_version; + BEUInt16_t m_revisionLevel; + BEUInt32_t m_vendor; + BEUInt32_t m_temporalQuality; + BEUInt32_t m_spatialQuality; + BEUInt16_t m_width; + BEUInt16_t m_height; + BEUFixed32_t m_hRes; + BEUFixed32_t m_vRes; + BEUInt32_t m_dataSize; + BEUInt16_t m_frameCountPerSample; + uint8_t m_compressorNameLength; + uint8_t m_compressorName[31]; + BEUInt16_t m_depth; + BEInt16_t m_ctabID; + + FieldAtom m_fieldAtom; + }; + + STSDEntry stsdEntry; + memset(&stsdEntry, 0, sizeof(stsdEntry)); + + stsdEntry.m_size = sizeof(STSDEntry) + sizeof(VideoMediaSampleDescription); + memcpy(stsdEntry.m_dataFormat, imageDesc.m_data.m_codecType, 4); + stsdEntry.m_dataRefIndex = 1; + + stream.Write(&stsdEntry, sizeof(stsdEntry)); + + const size_t kVideoMediaSampleDescSize = sizeof(VideoMediaSampleDescription); + GP_STATIC_ASSERT(kVideoMediaSampleDescSize == 80); + + VideoMediaSampleDescription vmsdDesc; + memset(&vmsdDesc, 0, sizeof(vmsdDesc)); + + memcpy(&vmsdDesc.m_vendor, "GR2A", 4); + vmsdDesc.m_temporalQuality = 0x200; + vmsdDesc.m_spatialQuality = 0x200; + vmsdDesc.m_width = imageDesc.m_data.m_width; + vmsdDesc.m_height = imageDesc.m_data.m_height; + vmsdDesc.m_hRes.m_intPart = 72; + vmsdDesc.m_vRes.m_intPart = 72; + vmsdDesc.m_frameCountPerSample = 1; + vmsdDesc.m_depth = imageDesc.m_data.m_depth; + vmsdDesc.m_ctabID = -1; + + memcpy(vmsdDesc.m_fieldAtom.m_atomID, "fild", 4); + vmsdDesc.m_fieldAtom.m_atomSize = sizeof(vmsdDesc.m_fieldAtom); + vmsdDesc.m_fieldAtom.m_fieldCount = 1; + + stream.Write(&vmsdDesc, sizeof(vmsdDesc)); + } + + { + MovDumpScope sttsScope(stream, "stts"); + + struct STTSData + { + uint8_t m_version; + uint8_t m_flags[3]; + BEUInt32_t m_numEntries; + }; + + STTSData sttsData; + memset(&sttsData, 0, sizeof(sttsData)); + sttsData.m_numEntries = 1; + + stream.Write(&sttsData, sizeof(sttsData)); + + struct STTSEntry + { + BEUInt32_t m_sampleCount; + BEUInt32_t m_sampleDuration; + }; + + STTSEntry sttsEntry; + memset(&sttsEntry, 0, sizeof(sttsEntry)); + sttsEntry.m_sampleCount = 1; + sttsEntry.m_sampleDuration = 0x200; + + stream.Write(&sttsEntry, sizeof(sttsEntry)); + } + + { + MovDumpScope sttsScope(stream, "stsc"); + + struct STSCData + { + uint8_t m_version; + uint8_t m_flags[3]; + BEUInt32_t m_numEntries; + }; + + STSCData stscData; + memset(&stscData, 0, sizeof(stscData)); + stscData.m_numEntries = 1; + + stream.Write(&stscData, sizeof(stscData)); + + struct STSCEntry + { + BEUInt32_t m_firstChunk; + BEUInt32_t m_samplesPerChunk; + BEUInt32_t m_sampleDescriptionID; + }; + + STSCEntry stscEntry; + memset(&stscEntry, 0, sizeof(stscEntry)); + stscEntry.m_firstChunk = 1; + stscEntry.m_samplesPerChunk = 1; + stscEntry.m_sampleDescriptionID = 1; + + stream.Write(&stscEntry, sizeof(stscEntry)); + } + + { + MovDumpScope sttsScope(stream, "stsz"); + + struct STSZData + { + uint8_t m_version; + uint8_t m_flags[3]; + BEUInt32_t m_sampleSize; + BEUInt32_t m_numEntries; + }; + + STSZData stszData; + memset(&stszData, 0, sizeof(stszData)); + stszData.m_sampleSize = static_cast(imageData.size()); + stszData.m_numEntries = 1; + + stream.Write(&stszData, sizeof(stszData)); + } + + { + MovDumpScope sttsScope(stream, "stco"); + + struct STCOData + { + uint8_t m_version; + uint8_t m_flags[3]; + BEUInt32_t m_numEntries; + + BEUInt32_t m_firstChunkOffset; + }; + + STCOData stcoData; + memset(&stcoData, 0, sizeof(stcoData)); + stcoData.m_firstChunkOffset = static_cast(imageDataStart); + stcoData.m_numEntries = 1; + + stream.Write(&stcoData, sizeof(stcoData)); + } + } + } + } + } + } + + return true; +} + +static bool ReadImageDesc(GpIOStream *stream, ImageDescription &imageDesc, std::vector *optAuxData) +{ + GpUFilePos_t imageDescStart = stream->Tell(); + if (!stream->ReadExact(&imageDesc.m_idSize, sizeof(imageDesc.m_idSize))) + return false; + + if (imageDesc.m_idSize < sizeof(ImageDescription)) + return false; + + if (!stream->ReadExact(&imageDesc.m_data, sizeof(imageDesc.m_data))) + return false; + + if (optAuxData == nullptr) + { + if (!stream->SeekStart(imageDescStart)) + return false; + + if (!stream->SeekCurrent(imageDesc.m_idSize)) + return false; + } + else + { + const size_t auxSize = imageDesc.m_idSize - sizeof(ImageDescription); + optAuxData->resize(auxSize); + if (auxSize > 0) + { + if (!stream->ReadExact(&(*optAuxData)[0], auxSize)) + return false; + } + } + + return true; +} + bool BMPDumperContext::EmitQTContent(GpIOStream *stream, uint32_t dataSize, bool isCompressed) { + GpUFilePos_t startPos = stream->Tell(); + + if (isCompressed && !m_dumpqtDir) + return false; + // Only one known house ("Magic" seems to use uncompressed, which is partly documented here: // https://github.com/gco/xee/blob/master/XeePhotoshopPICTLoader.m + struct UncompressedDataHeader + { + BEUInt16_t m_version; + BESFixed32_t m_matrix[9]; + BEUInt32_t m_matteSizeBytes; + BERect m_matteRect; + }; + + struct CompressedDataHeader + { + BEUInt16_t m_transferMode; + BERect m_srcRect; + BEUInt32_t m_preferredAccuracy; + BEUInt32_t m_maskSizeBytes; + }; + + UncompressedDataHeader dHeader; + if (!stream->ReadExact(&dHeader, sizeof(dHeader))) + return false; + + CompressedDataHeader cHeader; + ImageDescription matteDesc; + ImageDescription imageDesc; + std::vector imageDescAux; + + bool haveMDesc = false; + + if (isCompressed) + { + if (!stream->ReadExact(&cHeader, sizeof(cHeader))) + return false; + } + + if (dHeader.m_matteSizeBytes != 0) + { + haveMDesc = true; + if (!ReadImageDesc(stream, matteDesc, nullptr)) + return false; + } + + if (isCompressed) + { + if (cHeader.m_maskSizeBytes > 0) + { + if (!stream->SeekCurrent(cHeader.m_maskSizeBytes)) + return false; + } + + if (!ReadImageDesc(stream, imageDesc, &imageDescAux)) + return false; + + const size_t imageDataSize = imageDesc.m_data.m_dataSize; + + GpUFilePos_t imageDataPrefixSize = stream->Tell() - startPos; + if (imageDataPrefixSize > dataSize) + return false; + + const size_t dataAvailable = dataSize - imageDataPrefixSize; + + if (dataAvailable < imageDataSize) + return false; + + std::vector imageData; + imageData.resize(imageDataSize); + + if (!stream->ReadExact(&imageData[0], imageDataSize)) + return false; + + std::stringstream dumpPathStream; + dumpPathStream << m_dumpqtDir << "/" << m_resID << ".mov"; + + FILE *tempFile = fopen_utf8(dumpPathStream.str().c_str(), "wb"); + if (!tempFile) + return false; + + PortabilityLayer::CFileStream tempFileStream(tempFile); + bool dumpedOK = DumpMOV(tempFileStream, imageDesc, imageData); + tempFileStream.Close(); + + if (!dumpedOK) + return false; + + if (!stream->SeekStart(startPos)) + return false; + + if (!stream->SeekCurrent(dataSize)) + return false; + + return false; + } + else + { + BEUInt16_t subOpcode; + if (!stream->ReadExact(&subOpcode, sizeof(subOpcode))) + return false; + + int n = 0; + } + + // Known compressed cases and codecs: // "Egypt" res 10011: JPEG // "The Meadows" res 3000: Apple Video (a.k.a. Apple RPZA) @@ -682,10 +1390,10 @@ bool BMPDumperContext::Export(std::vector &outData) const return ExportBMP(m_frame.Width(), m_frame.Height(), m_pitchInElements, &m_pixelData[0], outData); } -bool ImportPICT(std::vector &outBMP, const void *inData, size_t inSize) +bool ImportPICT(std::vector &outBMP, const void *inData, size_t inSize, const char *dumpqtDir, int resID) { PortabilityLayer::MemReaderStream stream(inData, inSize); - BMPDumperContext context; + BMPDumperContext context(dumpqtDir, resID); PortabilityLayer::QDPictDecoder decoder; if (decoder.DecodePict(&stream, &context)) @@ -788,7 +1496,7 @@ bool ImportSound(std::vector &outWAV, const void *inData, size_t inSize { BEUInt32_t m_samplePtr; BEUInt32_t m_length; - BEFixed32_t m_sampleRate; + BEUFixed32_t m_sampleRate; BEUInt32_t m_loopStart; BEUInt32_t m_loopEnd; uint8_t m_encoding; @@ -799,7 +1507,7 @@ bool ImportSound(std::vector &outWAV, const void *inData, size_t inSize { BEUInt32_t m_samplePtr; BEUInt32_t m_channelCount; - BEFixed32_t m_sampleRate; + BEUFixed32_t m_sampleRate; BEUInt32_t m_loopStart; BEUInt32_t m_loopEnd; uint8_t m_encoding; @@ -819,7 +1527,7 @@ bool ImportSound(std::vector &outWAV, const void *inData, size_t inSize { BEUInt32_t m_samplePtr; BEUInt32_t m_channelCount; - BEFixed32_t m_sampleRate; + BEUFixed32_t m_sampleRate; BEUInt32_t m_loopStart; BEUInt32_t m_loopEnd; uint8_t m_encoding; @@ -969,7 +1677,7 @@ bool ImportSound(std::vector &outWAV, const void *inData, size_t inSize { fprintf(stderr, "WARNING: Downsampling sound resource %i to 8 bit\n", resID); - const size_t numSamples = extHeader.m_numSamples * extHeader.m_channelCount; + const size_t numSamples = extHeader.m_numSamples; resampledSound.resize(numSamples); for (size_t i = 0; i < numSamples; i++) @@ -999,6 +1707,100 @@ bool ImportSound(std::vector &outWAV, const void *inData, size_t inSize if (static_cast(header.m_sampleRate.m_fracPart) >= 0x8000) sampleRate++; + if (sampleRate == 0) + return false; + + std::vector rateChangedSound; + const uint32_t minSampleRate = 22000; + const uint32_t maxSampleRate = 23000; + if (sampleRate > maxSampleRate || sampleRate < minSampleRate) + { + uint32_t sampleRateRatioNumerator = 1; + uint32_t sampleRateRatioDenominator = 1; + + uint32_t targetSampleRate = 0; + if (sampleRate < minSampleRate) + { + targetSampleRate = sampleRate; + while (targetSampleRate * 2 <= maxSampleRate) + { + targetSampleRate *= 2; + sampleRateRatioNumerator *= 2; + } + } + else + { + targetSampleRate = sampleRate; + while (minSampleRate * 2 <= targetSampleRate) + { + targetSampleRate /= 2; + sampleRateRatioDenominator *= 2; + } + } + + if (targetSampleRate > maxSampleRate || targetSampleRate < minSampleRate) + { + targetSampleRate = 0x56ef; + sampleRateRatioNumerator = targetSampleRate; + sampleRateRatioDenominator = sampleRate; + } + + + targetSampleRate = sampleRate * sampleRateRatioNumerator / sampleRateRatioDenominator; + + fprintf(stderr, "WARNING: Resampling sound resource %i from %i to %i Hz\n", resID, static_cast(sampleRate), static_cast(targetSampleRate)); + + uint32_t postInterpolateSampleRateRatioNumerator = sampleRateRatioNumerator; + uint32_t postInterpolateSampleRateRatioDenominator = sampleRateRatioDenominator; + + std::vector interpolatableSound(sndBufferData, sndBufferData + inputDataLength); + while (postInterpolateSampleRateRatioNumerator * 2 <= postInterpolateSampleRateRatioDenominator) + { + size_t halfSize = interpolatableSound.size() / 2; + for (size_t i = 0; i < halfSize; i++) + interpolatableSound[i] = (interpolatableSound[i * 2] + interpolatableSound[i * 2 + 1] + 1) / 2; + interpolatableSound.resize(halfSize); + postInterpolateSampleRateRatioNumerator *= 2; + + if (postInterpolateSampleRateRatioNumerator % 2 == 0 && postInterpolateSampleRateRatioDenominator % 2 == 0) + { + postInterpolateSampleRateRatioNumerator /= 2; + postInterpolateSampleRateRatioDenominator /= 2; + } + } + + if (postInterpolateSampleRateRatioNumerator != postInterpolateSampleRateRatioDenominator) + { + size_t originalSize = interpolatableSound.size(); + size_t targetSize = originalSize * postInterpolateSampleRateRatioNumerator / postInterpolateSampleRateRatioDenominator; + rateChangedSound.resize(targetSize); + + for (size_t i = 0; i < targetSize; i++) + { + size_t sampleIndexTimes256 = i * postInterpolateSampleRateRatioDenominator * 256 / postInterpolateSampleRateRatioNumerator; + size_t startSampleIndex = sampleIndexTimes256 / 256; + size_t endSampleIndex = startSampleIndex + 1; + if (startSampleIndex >= originalSize) + startSampleIndex = originalSize - 1; + if (endSampleIndex >= originalSize) + endSampleIndex = originalSize - 1; + + size_t interpolation = sampleIndexTimes256 % 256; + uint8_t startSample = interpolatableSound[startSampleIndex]; + uint8_t endSample = interpolatableSound[endSampleIndex]; + + rateChangedSound[i] = ((startSample * (256 - interpolation) + endSample * interpolation + 128) / 256); + } + sampleRate = (sampleRate * sampleRateRatioNumerator * 2 + sampleRateRatioDenominator) / (sampleRateRatioDenominator * 2); + } + else + rateChangedSound = interpolatableSound; + + sndBufferData = &rateChangedSound[0]; + inputDataLength = rateChangedSound.size(); + outputDataLength = inputDataLength; + } + PortabilityLayer::WaveFormatChunkV1 formatChunk; const size_t riffTagSize = sizeof(PortabilityLayer::RIFFTag); @@ -1328,6 +2130,71 @@ void ReadFileToVector(FILE *f, std::vector &vec) } } +bool ParsePatchNames(const std::vector &patchFileContents, std::vector &names) +{ + rapidjson::Document document; + document.Parse(reinterpret_cast(&patchFileContents[0]), patchFileContents.size()); + if (document.HasParseError()) + { + fprintf(stderr, "Error occurred parsing patch data"); + fprintf(stderr, "Error code %i Location %i", static_cast(document.GetParseError()), static_cast(document.GetErrorOffset())); + return false; + } + + if (!document.IsObject()) + { + fprintf(stderr, "Patch document is not an object"); + return false; + } + + if (document.HasMember("add")) + { + const rapidjson::Value &addValue = document["add"]; + if (!addValue.IsObject()) + { + fprintf(stderr, "Patch add list is not an object"); + return false; + } + + for (rapidjson::Value::ConstMemberIterator it = addValue.MemberBegin(), itEnd = addValue.MemberEnd(); it != itEnd; ++it) + { + const rapidjson::Value &itemName = it->name; + if (!itemName.IsString()) + { + fprintf(stderr, "Patch add list item key is not a string"); + return false; + } + + const char *itemNameStr = itemName.GetString(); + names.push_back(std::string(itemNameStr)); + } + } + + if (document.HasMember("delete")) + { + const rapidjson::Value &deleteValue = document["delete"]; + if (!deleteValue.IsArray()) + { + fprintf(stderr, "Patch add list is not an object"); + return false; + } + + for (const rapidjson::Value *it = deleteValue.Begin(), *itEnd = deleteValue.End(); it != itEnd; ++it) + { + const rapidjson::Value &item = *it; + if (!item.IsString()) + { + fprintf(stderr, "Patch delete list item key is not a string"); + return false; + } + + names.push_back(std::string(item.GetString())); + } + } + + return true; +} + bool ApplyPatch(const std::vector &patchFileContents, std::vector &archive) { rapidjson::Document document; @@ -1461,7 +2328,16 @@ bool ApplyPatch(const std::vector &patchFileContents, std::vector &names, const char *name) +{ + for (const std::string &vname : names) + if (vname == name) + return true; + + return false; +} + +int ConvertSingleFile(const char *resPath, const PortabilityLayer::CombinedTimestamp &ts, FILE *patchF, const char *dumpqtDir, const char *outPath) { FILE *inF = fopen_utf8(resPath, "rb"); if (!inF) @@ -1519,6 +2395,14 @@ int ConvertSingleFile(const char *resPath, const PortabilityLayer::CombinedTimes { 16, 16, 8, PortabilityLayer::ResTypeID('ics8') }, }; + std::vector reservedNames; + + if (havePatchFile) + { + if (!ParsePatchNames(patchFileContents, reservedNames)) + return -1; + } + for (size_t tlIndex = 0; tlIndex < typeListCount; tlIndex++) { const PortabilityLayer::ResourceCompiledTypeList &typeList = typeLists[tlIndex]; @@ -1546,23 +2430,29 @@ int ConvertSingleFile(const char *resPath, const PortabilityLayer::CombinedTimes if (typeList.m_resType == pictTypeID || typeList.m_resType == dateTypeID) { - PlannedEntry entry; char resName[256]; sprintf_s(resName, "%s/%i.bmp", resTag.m_id, static_cast(res.m_resID)); + if (ContainsName(reservedNames, resName)) + continue; + + PlannedEntry entry; entry.m_name = resName; - if (ImportPICT(entry.m_uncompressedContents, resData, resSize)) + if (ImportPICT(entry.m_uncompressedContents, resData, resSize, dumpqtDir, res.m_resID)) contents.push_back(entry); else fprintf(stderr, "Failed to import PICT res %i\n", static_cast(res.m_resID)); } else if (typeList.m_resType == sndTypeID) { - PlannedEntry entry; char resName[256]; sprintf_s(resName, "%s/%i.wav", resTag.m_id, static_cast(res.m_resID)); + if (ContainsName(reservedNames, resName)) + continue; + + PlannedEntry entry; entry.m_name = resName; if (ImportSound(entry.m_uncompressedContents, resData, resSize, res.m_resID)) @@ -1572,10 +2462,13 @@ int ConvertSingleFile(const char *resPath, const PortabilityLayer::CombinedTimes } else if (typeList.m_resType == indexStringTypeID) { - PlannedEntry entry; char resName[256]; sprintf_s(resName, "%s/%i.txt", resTag.m_id, static_cast(res.m_resID)); + if (ContainsName(reservedNames, resName)) + continue; + + PlannedEntry entry; entry.m_name = resName; if (ImportIndexedString(entry.m_uncompressedContents, resData, resSize)) @@ -1583,10 +2476,13 @@ int ConvertSingleFile(const char *resPath, const PortabilityLayer::CombinedTimes } else if (typeList.m_resType == ditlTypeID) { - PlannedEntry entry; char resName[256]; sprintf_s(resName, "%s/%i.json", resTag.m_id, static_cast(res.m_resID)); + if (ContainsName(reservedNames, resName)) + continue; + + PlannedEntry entry; entry.m_name = resName; if (ImportDialogItemTemplate(entry.m_uncompressedContents, resData, resSize)) @@ -1601,26 +2497,35 @@ int ConvertSingleFile(const char *resPath, const PortabilityLayer::CombinedTimes const IconTypeSpec &iconSpec = iconTypeSpecs[i]; if (typeList.m_resType == iconSpec.m_resTypeID) { - isIcon = true; - - PlannedEntry entry; char resName[256]; sprintf_s(resName, "%s/%i.bmp", resTag.m_id, static_cast(res.m_resID)); - entry.m_name = resName; + if (!ContainsName(reservedNames, resName)) + { + isIcon = true; - if (ImportIcon(entry.m_uncompressedContents, resData, resSize, iconSpec.m_width, iconSpec.m_height, iconSpec.m_bpp)) - contents.push_back(entry); + PlannedEntry entry; + + entry.m_name = resName; + + if (ImportIcon(entry.m_uncompressedContents, resData, resSize, iconSpec.m_width, iconSpec.m_height, iconSpec.m_bpp)) + contents.push_back(entry); + + break; + } } } if (!isIcon) { - PlannedEntry entry; - char resName[256]; sprintf_s(resName, "%s/%i.bin", resTag.m_id, static_cast(res.m_resID)); + if (ContainsName(reservedNames, resName)) + continue; + + PlannedEntry entry; + entry.m_name = resName; entry.m_uncompressedContents.resize(res.GetSize()); @@ -1688,7 +2593,7 @@ int ConvertDirectory(const std::string &basePath, const PortabilityLayer::Combin fputs_utf8(houseArchivePath.c_str(), stdout); fprintf(stdout, "\n"); - int returnCode = ConvertSingleFile(resPath.c_str(), ts, nullptr, houseArchivePath.c_str()); + int returnCode = ConvertSingleFile(resPath.c_str(), ts, nullptr, nullptr, houseArchivePath.c_str()); if (returnCode) { fprintf(stderr, "An error occurred while converting\n"); @@ -1704,10 +2609,15 @@ int ConvertDirectory(const std::string &basePath, const PortabilityLayer::Combin int PrintUsage() { - fprintf(stderr, "Usage: gpr2gpa [patch.json]\n"); + fprintf(stderr, "Usage: gpr2gpa [options]\n"); fprintf(stderr, " gpr2gpa \\* \n"); fprintf(stderr, " gpr2gpa /* \n"); fprintf(stderr, " gpr2gpa * \n"); + fprintf(stderr, "\n"); + fprintf(stderr, "Options:\n"); + fprintf(stderr, " -patch patch.json\n"); + fprintf(stderr, " -dumpqt \n"); + return -1; } @@ -1744,19 +2654,43 @@ int toolMain(int argc, const char **argv) return ConvertDirectory(base.substr(0, base.length() - 2), ts); } - if (argc != 4 && argc != 5) - return PrintUsage(); - + const char *dumpqtPath = nullptr; FILE *patchF = nullptr; - if (argc == 5) + if (argc > 4) { - patchF = fopen_utf8(argv[4], "rb"); - if (!patchF) + for (int optArgIndex = 4; optArgIndex < argc; ) { - fprintf(stderr, "Error reading patch file"); - return -1; + const char *optArg = argv[optArgIndex++]; + if (!strcmp(optArg, "-patch")) + { + if (optArgIndex == argc) + return PrintUsage(); + + if (patchF != nullptr) + { + fprintf(stderr, "Already specified patch file"); + return -1; + } + + const char *patchPath = argv[optArgIndex++]; + patchF = fopen_utf8(patchPath, "rb"); + if (!patchF) + { + fprintf(stderr, "Error reading patch file"); + return -1; + } + } + else if (!strcmp(optArg, "-dumpqt")) + { + if (optArgIndex == argc) + return PrintUsage(); + + dumpqtPath = argv[optArgIndex++]; + } + else + return PrintUsage(); } } - return ConvertSingleFile(argv[1], ts, patchF, argv[3]); + return ConvertSingleFile(argv[1], ts, patchF, dumpqtPath, argv[3]); } diff --git a/unpacktool/StuffIt5Parser.cpp b/unpacktool/StuffIt5Parser.cpp index 943c9a9..4834253 100644 --- a/unpacktool/StuffIt5Parser.cpp +++ b/unpacktool/StuffIt5Parser.cpp @@ -6,6 +6,7 @@ #include "PLBigEndian.h" #include +#include #include "CSInputBuffer.h" @@ -101,9 +102,15 @@ struct StuffIt5Block std::vector m_filename; std::vector m_children; + int m_numChildren; - bool Read(IFileReader &reader) + int64_t m_endPos; + + bool Read(IFileReader &reader, bool &outIsDirectoryAppendage) { + outIsDirectoryAppendage = false; + + int64_t headerPos = reader.GetPosition(); if (!reader.ReadExact(&m_header, sizeof(m_header))) return false; @@ -145,13 +152,12 @@ struct StuffIt5Block if (commentLength > m_header.m_headerSize - sizeWithOnlyNameAndPasswordInfo - 4) return false; - m_commentSize = commentLength; m_commentPos = reader.GetPosition(); if (commentLength) { - if (reader.SeekCurrent(commentLength)) + if (!reader.SeekCurrent(commentLength)) return false; } @@ -166,6 +172,13 @@ struct StuffIt5Block if (!reader.SeekCurrent(m_header.m_headerSize - sizeWithCommentData)) return false; + if (m_header.m_dataForkDesc.m_uncompressedSize == static_cast(0xffffffff)) + { + outIsDirectoryAppendage = true; + m_endPos = reader.GetPosition(); + return true; + } + if (!reader.ReadExact(&m_annex1, sizeof(m_annex1))) return false; @@ -199,21 +212,13 @@ struct StuffIt5Block { int numFiles = (m_header.m_dataForkDesc.m_algorithm_dirNumFilesHigh << 8) | (m_header.m_dataForkDesc.m_passwordDataLength_dirNumFilesLow); - m_children.resize(numFiles); - for (int i = 0; i < numFiles; i++) - { - if (i != 0) - { - if (!reader.SeekStart(m_children[i - 1].m_header.m_nextEntryOffset)) - return false; - } - - if (!m_children[i].Read(reader)) - return false; - } + m_numChildren = numFiles; + m_endPos = reader.GetPosition(); } else { + m_numChildren = 0; + if (m_hasResourceFork) { m_resForkPos = reader.GetPosition(); @@ -221,6 +226,8 @@ struct StuffIt5Block } else m_dataForkPos = reader.GetPosition(); + + m_endPos = m_dataForkPos + m_header.m_dataForkDesc.m_compressedSize; } return true; @@ -304,6 +311,34 @@ bool StuffIt5Parser::Check(IFileReader &reader) return (*match) == '\0'; } +static bool RecursiveBuildTree(std::vector &dirBlocks, uint32_t dirPos, const std::vector &flatBlocks, const std::unordered_map &filePosToDirectoryBlock, const std::unordered_map &directoryBlockToFilePos, const std::unordered_map> &entryChildren, int depth) +{ + if (depth == 16) + return false; + + std::unordered_map>::const_iterator children = entryChildren.find(dirPos); + if (children == entryChildren.end()) + return true; + + for (size_t childIndex : children->second) + { + StuffIt5Block block = flatBlocks[childIndex]; + if (block.m_isDirectory) + { + std::unordered_map::const_iterator directoryFilePosIt = directoryBlockToFilePos.find(childIndex); + if (directoryFilePosIt == directoryBlockToFilePos.end()) + return false; + + if (!RecursiveBuildTree(block.m_children, directoryFilePosIt->second, flatBlocks, filePosToDirectoryBlock, directoryBlockToFilePos, entryChildren, depth + 1)) + return false; + } + + dirBlocks.push_back(static_cast(block)); + } + + return true; +} + ArchiveItemList *StuffIt5Parser::Parse(IFileReader &reader) { reader.SeekStart(0); @@ -317,17 +352,52 @@ ArchiveItemList *StuffIt5Parser::Parse(IFileReader &reader) if (!reader.SeekStart(header.m_rootDirFirstEntryOffset)) return nullptr; - std::vector rootDirBlocks; - rootDirBlocks.resize(numRootDirEntries); + size_t totalBlocks = numRootDirEntries; + std::vector flatBlocks; - for (int i = 0; i < numRootDirEntries; i++) + std::unordered_map directoryBlockToFilePos; + std::unordered_map filePosToDirectoryBlock; + + // Unfortunately StuffIt 5 archive next/prev entry chains seem to be meaningless. + // The only real way to determine directory structure is after the fact. + for (int i = 0; i < totalBlocks; i++) { - if (i != 0) - reader.SeekStart(rootDirBlocks[i - 1].m_header.m_nextEntryOffset); + int64_t fpos = reader.GetPosition(); - if (!rootDirBlocks[i].Read(reader)) + bool isAppendage = false; + StuffIt5Block flatBlock; + if (!flatBlock.Read(reader, isAppendage)) return nullptr; + + if (isAppendage) + { + totalBlocks++; + continue; + } + + if (flatBlock.m_isDirectory) + { + totalBlocks += flatBlock.m_numChildren; + directoryBlockToFilePos[flatBlocks.size()] = static_cast(fpos); + filePosToDirectoryBlock[static_cast(fpos)] = flatBlocks.size(); + } + + if (i != totalBlocks - 1) + { + if (!reader.SeekStart(flatBlock.m_endPos)) + return nullptr; + } + + flatBlocks.push_back(flatBlock); } + std::unordered_map> entryChildren; + + for (size_t i = 0; i < flatBlocks.size(); i++) + entryChildren[flatBlocks[i].m_header.m_dirEntryOffset].push_back(i); + + std::vector rootDirBlocks; + RecursiveBuildTree(rootDirBlocks, 0, flatBlocks, filePosToDirectoryBlock, directoryBlockToFilePos, entryChildren, 0); + return ConvertToItemList(rootDirBlocks); } diff --git a/unpacktool/unpacktool.cpp b/unpacktool/unpacktool.cpp index 630b3cb..e82a84b 100644 --- a/unpacktool/unpacktool.cpp +++ b/unpacktool/unpacktool.cpp @@ -95,7 +95,12 @@ StuffItParser g_stuffItParser; StuffIt5Parser g_stuffIt5Parser; CompactProParser g_compactProParser; -std::string LegalizeWindowsFileName(const std::string &path) +static bool IsSeparator(char c) +{ + return c == '/' || c == '\\'; +} + +std::string LegalizeWindowsFileName(const std::string &path, bool paranoid) { const size_t length = path.length(); @@ -115,6 +120,9 @@ std::string LegalizeWindowsFileName(const std::string &path) isLegalChar = false; } + if (paranoid && isLegalChar) + isLegalChar = c == '_' || c == ' ' || c == '.' || c == ',' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9'); + if (isLegalChar) legalizedPath.append(&c, 1); else @@ -209,7 +217,7 @@ void MakeIntermediateDirectories(const std::string &path) } } -int RecursiveExtractFiles(int depth, ArchiveItemList *itemList, const std::string &path, IFileReader &reader, const PortabilityLayer::CombinedTimestamp &ts); +int RecursiveExtractFiles(int depth, ArchiveItemList *itemList, const std::string &path, bool pathParanoid, IFileReader &reader, const PortabilityLayer::CombinedTimestamp &ts); int ExtractSingleFork(const ArchiveCompressedChunkDesc &chunkDesc, const std::string &path, IFileReader &reader) { @@ -361,7 +369,7 @@ int ExtractFile(const ArchiveItem &item, const std::string &path, IFileReader &r return 0; } -int ExtractItem(int depth, const ArchiveItem &item, const std::string &dirPath, IFileReader &reader, const PortabilityLayer::CombinedTimestamp &ts) +int ExtractItem(int depth, const ArchiveItem &item, const std::string &dirPath, bool pathParanoid, IFileReader &reader, const PortabilityLayer::CombinedTimestamp &ts) { std::string path(reinterpret_cast(item.m_fileNameUTF8.data()), item.m_fileNameUTF8.size()); @@ -371,7 +379,7 @@ int ExtractItem(int depth, const ArchiveItem &item, const std::string &dirPath, fputs_utf8(path.c_str(), stdout); printf("\n"); - path = LegalizeWindowsFileName(path); + path = LegalizeWindowsFileName(path, pathParanoid); path = dirPath + path; @@ -381,7 +389,7 @@ int ExtractItem(int depth, const ArchiveItem &item, const std::string &dirPath, path.append("\\"); - int returnCode = RecursiveExtractFiles(depth + 1, item.m_children, path, reader, ts); + int returnCode = RecursiveExtractFiles(depth + 1, item.m_children, path, pathParanoid, reader, ts); if (returnCode) return returnCode; @@ -391,14 +399,14 @@ int ExtractItem(int depth, const ArchiveItem &item, const std::string &dirPath, return ExtractFile(item, path, reader, ts); } -int RecursiveExtractFiles(int depth, ArchiveItemList *itemList, const std::string &path, IFileReader &reader, const PortabilityLayer::CombinedTimestamp &ts) +int RecursiveExtractFiles(int depth, ArchiveItemList *itemList, const std::string &path, bool pathParanoid, IFileReader &reader, const PortabilityLayer::CombinedTimestamp &ts) { const std::vector &items = itemList->m_items; const size_t numChildren = items.size(); for (size_t i = 0; i < numChildren; i++) { - int returnCode = ExtractItem(depth, items[i], path, reader, ts); + int returnCode = ExtractItem(depth, items[i], path, pathParanoid, reader, ts); if (returnCode) return returnCode; } @@ -406,22 +414,25 @@ int RecursiveExtractFiles(int depth, ArchiveItemList *itemList, const std::strin return 0; } -int toolMain(int argc, const char **argv) +int PrintUsage() { - if (argc != 4) - { - fprintf(stderr, "Usage: unpacktool "); - return -1; - } + fprintf(stderr, "Usage: unpacktool [options]"); + fprintf(stderr, "Usage: unpacktool -bulk "); + return -1; +} - FILE *inputArchive = fopen_utf8(argv[1], "rb"); +int decompMain(int argc, const char **argv) +{ + for (int i = 0; i < argc; i++) + printf("%s\n", argv[i]); - if (!inputArchive) - { - fprintf(stderr, "Could not open input archive"); - return -1; - } + if (argc < 4) + return PrintUsage(); + bool isBulkMode = !strcmp(argv[1], "-bulk"); + + if (!isBulkMode && argc < 4) + return PrintUsage(); FILE *tsFile = fopen_utf8(argv[2], "rb"); @@ -440,45 +451,111 @@ int toolMain(int argc, const char **argv) fclose(tsFile); - CFileReader reader(inputArchive); + int arcArg = 1; + int numArgArcs = 1; - IArchiveParser *parsers[] = + if (isBulkMode) { - &g_compactProParser, - &g_stuffItParser, - &g_stuffIt5Parser - }; + arcArg = 3; + numArgArcs = argc - 3; + } - ArchiveItemList *archiveItemList = nullptr; - - printf("Reading archive...\n"); - - for (IArchiveParser *parser : parsers) + bool pathParanoid = false; + if (!isBulkMode) { - if (parser->Check(reader)) + for (int optArgIndex = 4; optArgIndex < argc; ) { - archiveItemList = parser->Parse(reader); - break; + const char *optArg = argv[optArgIndex++]; + + if (!strcmp(optArg, "-paranoid")) + pathParanoid = true; + else + { + fprintf(stderr, "Unknown option %s\n", optArg); + return -1; + } } } - if (!archiveItemList) + for (int arcArgIndex = 0; arcArgIndex < numArgArcs; arcArgIndex++) { - fprintf(stderr, "Failed to open archive"); - return -1; + const char *arcPath = argv[arcArg + arcArgIndex]; + + FILE *inputArchive = fopen_utf8(arcPath, "rb"); + + std::string destPath; + if (isBulkMode) + { + destPath = arcPath; + size_t lastSepIndex = 0; + for (size_t i = 1; i < destPath.size(); i++) + { + if (destPath[i] == '/' || destPath[i] == '\\') + lastSepIndex = i; + } + + destPath = destPath.substr(0, lastSepIndex); + } + else + destPath = argv[3]; + + if (!inputArchive) + { + fprintf(stderr, "Could not open input archive"); + return -1; + } + + CFileReader reader(inputArchive); + + IArchiveParser *parsers[] = + { + &g_compactProParser, + &g_stuffItParser, + &g_stuffIt5Parser + }; + + ArchiveItemList *archiveItemList = nullptr; + + printf("Reading archive '%s'...\n", arcPath); + + for (IArchiveParser *parser : parsers) + { + if (parser->Check(reader)) + { + archiveItemList = parser->Parse(reader); + break; + } + } + + if (!archiveItemList) + { + fprintf(stderr, "Failed to open archive"); + return -1; + } + + printf("Decompressing files...\n"); + + std::string currentPath = destPath; + TerminateDirectoryPath(currentPath); + + MakeIntermediateDirectories(currentPath); + + int returnCode = RecursiveExtractFiles(0, archiveItemList, currentPath, pathParanoid, reader, ts); + if (returnCode != 0) + { + fprintf(stderr, "Error decompressing archive"); + return returnCode; + } + + delete archiveItemList; } - printf("Decompressing files...\n"); - - std::string currentPath = argv[3]; - TerminateDirectoryPath(currentPath); - - MakeIntermediateDirectories(currentPath); - - int returnCode = RecursiveExtractFiles(0, archiveItemList, currentPath, reader, ts); - - delete archiveItemList; - - return returnCode; + return 0; } + +int toolMain(int argc, const char **argv) +{ + int returnCode = decompMain(argc, argv); + return returnCode; +}