Fix mac frozen dependencies (@rpath, /usr/local/) to use @executable_path, and repair the broken cx_Freeze folder and *.app folder. Also fix various dmg building errors due to new file path locations.

This commit is contained in:
Jonathan Thomas
2021-01-23 18:37:04 -06:00
parent 2f48b0089a
commit 1358dc0dd4
5 changed files with 148 additions and 123 deletions
+2 -2
View File
@@ -63,8 +63,8 @@ mac-builder:
- export DYLD_LIBRARY_PATH=$CI_PROJECT_DIR/build/install-x64/lib:DYLD_LIBRARY_PATH
- echo -e "CI_PROJECT_NAME:$CI_PROJECT_NAME\nCI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME\nCI_COMMIT_SHA:$CI_COMMIT_SHA\nCI_JOB_ID:$CI_JOB_ID\nCI_PIPELINE_ID:$CI_PIPELINE_ID" > "build/install-x64/share/$CI_PROJECT_NAME"
- git log $(git describe --tags --abbrev=0 @^)..@ --oneline --pretty=format:"%C(auto,yellow)%h%C(auto,magenta)% %C(auto,blue)%>(12,trunc)%ad %C(auto,green)%<(25,trunc)%aN%C(auto,reset)%s%C(auto,red)% gD% D" --date=short > "build/install-x64/share/$CI_PROJECT_NAME.log"
- /Library/Frameworks/Python.Framework/Versions/3.6/bin/python3.6 -u freeze.py bdist_mac --iconfile=installer/openshot.icns --custom-info-plist=installer/Info.plist --bundle-name="OpenShot Video Editor"
- /Library/Frameworks/Python.Framework/Versions/3.6/bin/python3.6 -u installer/build-server.py "$SLACK_TOKEN" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" "$WINDOWS_KEY" "$WINDOWS_PASSWORD" "$GITHUB_USER" "$GITHUB_PASS" "False" "$CI_COMMIT_REF_NAME"
- python3 -u freeze.py bdist_mac --iconfile=installer/openshot.icns --custom-info-plist=installer/Info.plist --bundle-name="OpenShot Video Editor"
- python3 -u installer/build-server.py "$SLACK_TOKEN" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" "$WINDOWS_KEY" "$WINDOWS_PASSWORD" "$GITHUB_USER" "$GITHUB_PASS" "False" "$CI_COMMIT_REF_NAME"
when: always
except:
- tags
+35 -12
View File
@@ -195,6 +195,11 @@ if sys.platform == "win32":
for filename in find_files(zmq_path, ["*"]):
src_files.append((filename, os.path.join("lib", "zmq", os.path.relpath(filename, start=zmq_path))))
# Append all source files
src_files.append((os.path.join(PATH, "installer", "qt.conf"), "qt.conf"))
for filename in find_files("openshot_qt", ["*"]):
src_files.append((filename, os.path.join(os.path.relpath(filename, start=os.path.join(PATH, "openshot_qt")))))
elif sys.platform == "linux":
# Find libopenshot.so path (GitLab copies artifacts into local build/install folder)
libopenshot_path = os.path.join(PATH, "build", "install-x64", "lib")
@@ -348,19 +353,19 @@ elif sys.platform == "linux":
# Any other lib deps that fail to meet the inclusion
# criteria above will be silently skipped over
# Append all source files
src_files.append((os.path.join(PATH, "installer", "qt.conf"), "qt.conf"))
for filename in find_files("openshot_qt", ["*"]):
src_files.append((filename, os.path.join(os.path.relpath(filename, start=os.path.join(PATH, "openshot_qt")))))
elif sys.platform == "darwin":
# Copy Mac specific files that cx_Freeze misses
# JPEG library
for filename in find_files("/usr/local/Cellar/jpeg/8d/lib", ["libjpeg.8.dylib"]):
external_so_files.append((filename, filename.replace("/usr/local/Cellar/jpeg/8d/lib/", "")))
# Add libresvg (if found)
resvg_path = "/usr/local/lib/libresvg.dylib"
# Add libresvg (if found)
resvg_path = "/usr/local/lib/librsvg-2.dylib"
if os.path.exists(resvg_path):
external_so_files.append((resvg_path, resvg_path.replace("/usr/local/lib/", "")))
# Copy openshot.py Python bindings
src_files.append((os.path.join(PATH, "openshot.py"), "openshot.py"))
src_files.append((os.path.join(PATH, "installer", "launch-mac"), "launch-mac"))
# Append Mac ICON file
@@ -383,18 +388,23 @@ elif sys.platform == "darwin":
for filename in find_files(os.path.join(qt_install_path, "plugins"), ["*"]):
relative_filepath = os.path.relpath(filename, start=os.path.join(qt_install_path, "plugins"))
plugin_name = os.path.dirname(relative_filepath)
if plugin_name not in ["sqldrivers", "playlistformats", "gamepads", "bearer"]:
if plugin_name in ["imageformats", "platforms"]:
external_so_files.append((filename, relative_filepath))
# Append all source files
src_files.append((os.path.join(PATH, "installer", "qt.conf"), "qt.conf"))
for filename in find_files("openshot_qt", ["*"]):
src_files.append((filename, filename.replace("openshot_qt/", "").replace("openshot_qt\\", "")))
# Append all source files
src_files.append((os.path.join(PATH, "installer", "qt.conf"), "qt.conf"))
for filename in find_files("openshot_qt", ["*"]):
src_files.append((filename, os.path.join("lib", os.path.relpath(filename, start=os.path.join(PATH, "openshot_qt")))))
# Exclude gif library which crashes on Mac
build_exe_options["bin_excludes"] = ["/System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libGIF.dylib",
"/usr/local/opt/giflib/lib/libgif.dylib"]
# Dependencies are automatically detected, but it might need fine tuning.
build_exe_options["packages"] = python_packages
build_exe_options["include_files"] = src_files + external_so_files
build_exe_options["includes"] = python_modules
build_exe_options["excludes"] = ["distutils", "numpy", "setuptools", "tkinter", "pydoc_data", "pycparser", "pkg_resources"]
# Set options
build_options["build_exe"] = build_exe_options
@@ -428,3 +438,16 @@ setup(name=info.PRODUCT_NAME,
# Remove temporary folder (if SRC folder present)
if os.path.exists(os.path.join(PATH, "src")):
rmtree(os.path.join(PATH, "openshot_qt"), True)
# Fix a few things on the frozen folder(s)
if sys.platform == "darwin":
# Mac issues with frozen folder and *.app folder
# We need to rewrite many dependency paths and library IDs
from installer.fix_qt5_rpath import *
build_path = os.path.join(PATH, "build")
for frozen_path in os.listdir(build_path):
if frozen_path.startswith("exe"):
fix_rpath(os.path.join(build_path, frozen_path))
elif frozen_path.endswith(".app"):
fix_rpath(os.path.join(build_path, frozen_path, "Contents", "MacOS"))
print_min_versions(os.path.join(build_path, frozen_path, "Contents", "MacOS"))
View File
+17 -28
View File
@@ -1,5 +1,5 @@
#!/bin/sh
PATH=/Library/Frameworks/Python.framework/Versions/3.6/bin:/opt/local/bin:/opt/local/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/qt5/5.5/clang_64/bin:/opt/X11/bin
PATH=/usr/local/Cellar/python@3.7/3.7.9_2/Frameworks/Python.framework/Versions/3.7/bin:/opt/local/bin:/opt/local/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/qt5/5.5/clang_64/bin:/opt/X11/bin
# Get Version
VERSION=$(grep -E '^VERSION = "(.*)"' src/classes/info.py | awk '{print $3}' | tr -d '"')
@@ -16,37 +16,26 @@ cp installer/Info.plist "$OS_PATH"
sed -e "s/VERSION/$VERSION/g" "$OS_PATH/Info.plist" > "$OS_PATH/Info.plist_version"
mv "$OS_PATH/Info.plist_version" "$OS_PATH/Info.plist"
if [ ! -d "$OS_PATH/MacOS/lib" ]; then
echo "Creating lib folder"
mkdir "$OS_PATH/MacOS/lib"
fi
echo "Symlink Non-Code Files to Resources"
mv "$OS_PATH/MacOS/blender" "$OS_PATH/Resources/blender"; ln -s "../Resources/blender" "$OS_PATH/MacOS/blender";
mv "$OS_PATH/MacOS/classes" "$OS_PATH/Resources/classes"; ln -s "../Resources/classes" "$OS_PATH/MacOS/classes";
mv "$OS_PATH/MacOS/effects" "$OS_PATH/Resources/effects"; ln -s "../Resources/effects" "$OS_PATH/MacOS/effects";
mv "$OS_PATH/MacOS/emojis" "$OS_PATH/Resources/emojis"; ln -s "../Resources/emojis" "$OS_PATH/MacOS/emojis";
mv "$OS_PATH/MacOS/images" "$OS_PATH/Resources/images"; ln -s "../Resources/images" "$OS_PATH/MacOS/images";
mv "$OS_PATH/MacOS/language" "$OS_PATH/Resources/language"; ln -s "../Resources/language" "$OS_PATH/MacOS/language";
mv "$OS_PATH/MacOS/lib/blender" "$OS_PATH/Resources/blender"; ln -s "../../Resources/blender" "$OS_PATH/MacOS/lib/blender";
mv "$OS_PATH/MacOS/lib/classes" "$OS_PATH/Resources/classes"; ln -s "../../Resources/classes" "$OS_PATH/MacOS/lib/classes";
mv "$OS_PATH/MacOS/lib/effects" "$OS_PATH/Resources/effects"; ln -s "../../Resources/effects" "$OS_PATH/MacOS/lib/effects";
mv "$OS_PATH/MacOS/lib/emojis" "$OS_PATH/Resources/emojis"; ln -s "../../Resources/emojis" "$OS_PATH/MacOS/lib/emojis";
mv "$OS_PATH/MacOS/lib/images" "$OS_PATH/Resources/images"; ln -s "../../Resources/images" "$OS_PATH/MacOS/lib/images";
mv "$OS_PATH/MacOS/lib/language" "$OS_PATH/Resources/language"; ln -s "../../Resources/language" "$OS_PATH/MacOS/lib/language";
mv "$OS_PATH/MacOS/qtwebengine_locales" "$OS_PATH/Resources/qtwebengine_locales"; ln -s "../Resources/qtwebengine_locales" "$OS_PATH/MacOS/qtwebengine_locales";
mv "$OS_PATH/MacOS/presets" "$OS_PATH/Resources/presets"; ln -s "../Resources/presets" "$OS_PATH/MacOS/presets";
mv "$OS_PATH/MacOS/profiles" "$OS_PATH/Resources/profiles"; ln -s "../Resources/profiles" "$OS_PATH/MacOS/profiles";
mv "$OS_PATH/MacOS/resources" "$OS_PATH/Resources/resources"; ln -s "../Resources/resources" "$OS_PATH/MacOS/resources";
mv "$OS_PATH/MacOS/settings" "$OS_PATH/Resources/settings"; ln -s "../Resources/settings" "$OS_PATH/MacOS/settings";
mv "$OS_PATH/MacOS/tests" "$OS_PATH/Resources/tests"; ln -s "../Resources/tests" "$OS_PATH/MacOS/tests";
mv "$OS_PATH/MacOS/timeline" "$OS_PATH/Resources/timeline"; ln -s "../Resources/timeline" "$OS_PATH/MacOS/timeline";
mv "$OS_PATH/MacOS/titles" "$OS_PATH/Resources/titles"; ln -s "../Resources/titles" "$OS_PATH/MacOS/titles";
mv "$OS_PATH/MacOS/transitions" "$OS_PATH/Resources/transitions"; ln -s "../Resources/transitions" "$OS_PATH/MacOS/transitions";
mv "$OS_PATH/MacOS/windows" "$OS_PATH/Resources/windows"; ln -s "../Resources/windows" "$OS_PATH/MacOS/windows";
mv "$OS_PATH/MacOS/lib/presets" "$OS_PATH/Resources/presets"; ln -s "../../Resources/presets" "$OS_PATH/MacOS/lib/presets";
mv "$OS_PATH/MacOS/lib/profiles" "$OS_PATH/Resources/profiles"; ln -s "../../Resources/profiles" "$OS_PATH/MacOS/lib/profiles";
mv "$OS_PATH/MacOS/lib/resources" "$OS_PATH/Resources/resources"; ln -s "../../Resources/resources" "$OS_PATH/MacOS/lib/resources";
mv "$OS_PATH/MacOS/lib/settings" "$OS_PATH/Resources/settings"; ln -s "../../Resources/settings" "$OS_PATH/MacOS/lib/settings";
mv "$OS_PATH/MacOS/lib/tests" "$OS_PATH/Resources/tests"; ln -s "../../Resources/tests" "$OS_PATH/MacOS/lib/tests";
mv "$OS_PATH/MacOS/lib/timeline" "$OS_PATH/Resources/timeline"; ln -s "../../Resources/timeline" "$OS_PATH/MacOS/lib/timeline";
mv "$OS_PATH/MacOS/lib/titles" "$OS_PATH/Resources/titles"; ln -s "../../Resources/titles" "$OS_PATH/MacOS/lib/titles";
mv "$OS_PATH/MacOS/lib/transitions" "$OS_PATH/Resources/transitions"; ln -s "../../Resources/transitions" "$OS_PATH/MacOS/lib/transitions";
mv "$OS_PATH/MacOS/lib/windows" "$OS_PATH/Resources/windows"; ln -s "../../Resources/windows" "$OS_PATH/MacOS/lib/windows";
mv "$OS_PATH/MacOS/qt.conf" "$OS_PATH/Resources/qt.conf"; ln -s "../Resources/qt.conf" "$OS_PATH/MacOS/qt.conf";
mv "$OS_PATH/MacOS/openshot.py" "$OS_PATH/Resources/openshot.py"; ln -s "../Resources/openshot.py" "$OS_PATH/MacOS/openshot.py";
mv "$OS_PATH/MacOS/openshot-qt.hqx" "$OS_PATH/Resources/openshot-qt.hqx"; ln -s "../Resources/openshot-qt.hqx" "$OS_PATH/MacOS/openshot-qt.hqx";
mv "$OS_PATH/MacOS/launch.py" "$OS_PATH/Resources/launch.py"; ln -s "../Resources/launch.py" "$OS_PATH/MacOS/launch.py";
if [ -d "$OS_PATH/MacOS/python3.6" ]; then
echo "Symlink python36.zip and python3.6 folder"
mv "$OS_PATH/MacOS/python3.6" "$OS_PATH/Resources/python3.6"; ln -s "../../Resources/python3.6" "$OS_PATH/MacOS/lib/python3.6";
mv "$OS_PATH/MacOS/python36.zip" "$OS_PATH/Resources/python36.zip"; ln -s "../../Resources/python36.zip" "$OS_PATH/MacOS/lib/python36.zip";
fi
mv "$OS_PATH/MacOS/lib/launch.py" "$OS_PATH/Resources/launch.py"; ln -s "../../Resources/launch.py" "$OS_PATH/MacOS/lib/launch.py";
echo "Loop through bundled files and sign all binary files"
find "build" \( -iname '*.dylib' -o -iname '*.so' \) -exec codesign -s "OpenShot Studios, LLC" --timestamp=http://timestamp.apple.com/ts01 --entitlements "installer/openshot.entitlements" --force "{}" \;
+94 -81
View File
@@ -1,90 +1,103 @@
"""Analyze the OS X app bundle for OpenShot, and find all external dependencies"""
"""Methods to fix Mac OS X @rpath and /usr/local dependencies, and analyze the
app bundle for OpenShot to find all external dependencies"""
import os
import subprocess
from subprocess import call
import re
# Symbolic Link Qt Frameworks into Python3/PyQt5 Framework
# IMPORTANT to run after installing PyQt5
PATH = "/usr/local/qt5.15.X/qt5.15/5.15.0/clang_64/lib"
# Find files matching patterns
for file in os.listdir(PATH):
if file.endswith(".framework"):
file_path = os.path.join(PATH, file)
file_parts = file_path.split("/")
new_path = os.path.join("/Library/Frameworks/Python.framework/Versions/3.6/lib", file_parts[-1])
print(new_path)
if os.path.exists(new_path):
# Remove old sym links
os.unlink(new_path)
# Create new symlink to Qt frameworks
call(["ln", "-s", file_path, new_path])
# Symbolic Link Qt Frameworks into Python3/PyQt5 Framework
# IMPORTANT to run after installing PyQt5
PATH = "/usr/local/qt5.15.X/qt5.15/5.15.0/clang_64/lib"
# Find files matching patterns
for file in os.listdir(PATH):
if file.endswith(".framework"):
file_path = os.path.join(PATH, file)
file_parts = file_path.split("/")
new_path = os.path.join("/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/PyQt5/lib", file_parts[-1])
print(new_path)
if os.path.exists(new_path):
# Remove old sym links
os.unlink(new_path)
# Create new symlink to Qt frameworks
call(["ln", "-s", file_path, new_path])
import shutil
# FIX PyQt5 library paths
# This is NOT required anymore (leaving here as an example)
#PATH = "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/PyQt5/"
# Ignore non library/executable extensions
non_executables = [".py", ".svg", ".png", ".blend", ".a", ".pak", ".qm", ".pyc", ".txt",
".jpg", ".zip", ".dat", ".conf", ".xml", ".h", ".ui", ".json", ".exe"]
# Find files matching patterns
#for root, dirs, files in os.walk(PATH):
# for basename in files:
# file_path = os.path.join(root, basename)
# file_parts = file_path.split("/")
#
# if ".dylib" in file_path or ".so" in file_path:
#
# #print (file_path)
# print ("install_name_tool %s -id %s" % (file_path, file_path))
# call(["install_name_tool", file_path, "-id", file_path])
#
# raw_output = subprocess.Popen(["oTool", "-L", file_path], stdout=subprocess.PIPE).communicate()[0].decode('utf-8')
# for output in raw_output.split("\n")[1:-1]:
# if output and not "is not an object file" in output:
# dependency_path = output.split('\t')[1].split(' ')[0]
# dependency_version = output.split('\t')[1].split(' (')[1].replace(')','')
#
# if "@rpath" in dependency_path:
# command = 'install_name_tool "%s" -change "%s" "%s"' % (file_path, dependency_path, dependency_path.replace("@rpath", os.path.join(PATH, "lib")))
# print (command)
# call(["install_name_tool", file_path, "-change", dependency_path, dependency_path.replace("@rpath", os.path.join(PATH, "lib"))])
def fix_rpath(PATH):
"""FIX broken @rpath on Qt, PyQt, and /usr/local/ dependencies with no @rpath"""
# Cleanup duplicate folder and files (that cx_freeze creates)
duplicate_path = os.path.join(PATH, "lib", "openshot_qt")
if os.path.exists(duplicate_path):
shutil.rmtree(duplicate_path)
# Find files matching patterns
for root, dirs, files in os.walk(PATH):
for basename in files:
file_path = os.path.join(root, basename)
relative_path = os.path.relpath(root, PATH)
# Skip common file extensions (speed things up)
if os.path.splitext(file_path)[-1] in non_executables or basename.startswith("."):
continue
# Build exe path (relative from the app dir)
executable_path = os.path.join("@executable_path", relative_path, basename)
if relative_path == ".":
executable_path = os.path.join("@executable_path", basename)
# Change ID path of library files
# Sometimes, the ID has an absolute path or invalid @rpath embedded in it, which breaks our frozen exe
print ("ID: %s: install_name_tool %s -id %s" % (basename, file_path, executable_path))
call(["install_name_tool", file_path, "-id", executable_path])
# Loop through all dependencies of each library/executable
raw_output = subprocess.Popen(["oTool", "-L", file_path], stdout=subprocess.PIPE).communicate()[0].decode('utf-8')
for output in raw_output.split("\n")[1:-1]:
if output and not "is not an object file" in output and ".o):" not in output:
dependency_path = output.split('\t')[1].split(' ')[0]
dependency_version = output.split('\t')[1].split(' (')[1].replace(')', '')
dependency_base_path, dependency_name = os.path.split(dependency_path)
# If @rpath or /usr/local found in dependency path, update with @executable_path instead
if "/usr/local" in dependency_path or "@rpath" in dependency_path:
dependency_exe_path = os.path.join("@executable_path", dependency_name)
if not os.path.exists(os.path.join(PATH, dependency_name)):
print("ERROR: /usr/local PATH not found in EXE folder: %s" % dependency_path)
else:
print('-- Dependency: %s: install_name_tool "%s" -change "%s" "%s"' % (dependency_name, file_path, dependency_path, dependency_exe_path))
call(["install_name_tool", file_path, "-change", dependency_path, dependency_exe_path])
# Print ALL MINIMUM and SDK VERSIONS for files in OpenShot build folder
# This does not list all dependent libraries though, and sometimes one of those can cause issues.
# PATH = "/Users/jonathan/builds/7d5103a1/0/OpenShot/openshot-qt/build/exe.macosx-10.6-intel-3.6"
# REGEX_SDK_MATCH = re.compile(r'.*(LC_VERSION_MIN_MACOSX).*version (\d+\.\d+).*sdk (\d+\.\d+).*(cmd)', re.DOTALL)
#
# # Find files matching patterns
# for root, dirs, files in os.walk(PATH):
# for basename in files:
# file_path = os.path.join(root, basename)
# file_parts = file_path.split("/")
#
# raw_output = subprocess.Popen(["oTool", "-l", file_path], stdout=subprocess.PIPE).communicate()[0].decode('utf-8')
# matches = REGEX_SDK_MATCH.findall(raw_output)
# if matches and len(matches[0]) == 4:
# min_version = matches[0][1]
# sdk_version = matches[0][2]
# print("min: %s\tsdk: %s\t%s" % (min_version, sdk_version, file_path))
# if min_version in ['10.14', '10.15']:
# print("ERROR!!!! Minimum OS X version not met for %s" % file_path)
def print_min_versions(PATH):
"""Print ALL MINIMUM and SDK VERSIONS for files in OpenShot build folder.
This does not list all dependent libraries though, and sometimes one of those can cause issues."""
REGEX_SDK_MATCH = re.compile(r'.*(LC_VERSION_MIN_MACOSX).*version (\d+\.\d+).*sdk (\d+\.\d+).*(cmd)', re.DOTALL)
VERSIONS = {}
# Find files matching patterns
for root, dirs, files in os.walk(PATH):
for basename in files:
file_path = os.path.join(root, basename)
file_parts = file_path.split("/")
# Skip common file extensions (speed things up)
if os.path.splitext(file_path)[-1] in non_executables or basename.startswith("."):
continue
raw_output = subprocess.Popen(["oTool", "-l", file_path], stdout=subprocess.PIPE).communicate()[0].decode('utf-8')
matches = REGEX_SDK_MATCH.findall(raw_output)
if matches and len(matches[0]) == 4:
min_version = matches[0][1]
sdk_version = matches[0][2]
print("... scanning %s for min version (min: %s, sdk: %s)" % (file_path.replace(PATH, ""),
min_version, sdk_version))
# Organize by MIN version
if min_version in VERSIONS:
if file_path not in VERSIONS[min_version]:
VERSIONS[min_version].append(file_path)
else:
VERSIONS[min_version] = [file_path]
if min_version in ['10.14', '10.15']:
print("ERROR!!!! Minimum OS X version not met for %s" % file_path)
print("\nSummary of Minimum Mac SDKs for Dependencies:")
for key in sorted(VERSIONS.keys()):
print("\n%s" % key)
for file_path in VERSIONS[key]:
print(" %s" % file_path)
if __name__ == "__main__":
"""Run these methods manually for testing"""
PATH = "/Users/jonathanthomas/apps/openshot-qt/build/exe.macosx-10.15-x86_64-3.7"
fix_rpath(PATH)
print_min_versions(PATH)