diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2c6e521175420f24fa4e8772cf7d74c986f8c224..27cb2d651a9572e37609ea6e49bb2659bb618532 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -47,7 +47,7 @@ pylint:
before_script:
- pip install pylint
script:
- - pylint $PWD/pyotb
+ - pylint $PWD/pyotb --disable=fixme
codespell:
extends: .static_analysis
@@ -84,7 +84,7 @@ test_install:
before_script:
- pip install pytest pytest-cov
-test_module_core:
+module_core:
extends: .tests
variables:
OTB_LOGGER_LEVEL: INFO
@@ -101,7 +101,7 @@ test_module_core:
- curl -fsLI $PLEIADES_IMG_URL
- python3 -m pytest -vv --junitxml=test-module-core.xml --cov-report xml:coverage.xml tests/test_core.py
-test_pipeline_permutations:
+pipeline_permutations:
extends: .tests
variables:
OTB_LOGGER_LEVEL: WARNING
diff --git a/pyotb/__init__.py b/pyotb/__init__.py
index bf553928bd558a37600a3d6bb50796dcc64e0fda..aff3736392ba7e0d464110dccbba05063b854b38 100644
--- a/pyotb/__init__.py
+++ b/pyotb/__init__.py
@@ -1,10 +1,19 @@
# -*- coding: utf-8 -*-
"""This module provides convenient python wrapping of otbApplications."""
-__version__ = "2.0.0.dev4"
+__version__ = "2.0.0.dev6"
+from .install import install_otb
from .helpers import logger, set_logger_level
+from .core import (
+ OTBObject,
+ App,
+ Input,
+ Output,
+ get_nbchannels,
+ get_pixel_type,
+ summarize,
+)
from .apps import *
-from .core import App, Input, Output, get_nbchannels, get_pixel_type, summarize, OTBObject
from .functions import ( # pylint: disable=redefined-builtin
all,
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index 5363937940485b33f74c31ef8371af46279e2e56..3fc93b389fde634b0e81f24ae20a81fa74eb8776 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -1,13 +1,17 @@
# -*- coding: utf-8 -*-
-"""This module helps to ensure we properly initialize pyotb: only in case OTB is found and apps are available."""
+"""This module ensure we properly initialize pyotb, or raise SystemExit in case of broken install."""
import logging
import os
import sys
+import sysconfig
from pathlib import Path
from shutil import which
+from .install import install_otb, interactive_config
+
# Allow user to switch between OTB directories without setting every env variable
OTB_ROOT = os.environ.get("OTB_ROOT")
+DOCS_URL = "https://www.orfeo-toolbox.org/CookBook/Installation.html"
# Logging
# User can also get logger with `logging.getLogger("pyOTB")`
@@ -39,19 +43,20 @@ def set_logger_level(level: str):
logger_handler.setLevel(getattr(logging, level))
-def find_otb(prefix: str = OTB_ROOT, scan: bool = True, scan_userdir: bool = True):
- """Try to load OTB bindings or scan system, help user in case of failure, set env variables.
+def find_otb(prefix: str = OTB_ROOT, scan: bool = True):
+ """Try to load OTB bindings or scan system, help user in case of failure, set env.
- Path precedence : OTB_ROOT > python bindings directory
- OR search for releases installations : HOME
- OR (for Linux) : /opt/otbtf > /opt/otb > /usr/local > /usr
- OR (for MacOS) : ~/Applications
- OR (for Windows) : C:/Program Files
+ If in interactive prompt, user will be asked if he wants to install OTB.
+ The OTB_ROOT variable allow one to override default OTB version, with auto env setting.
+ Path precedence : $OTB_ROOT > location of python bindings location
+ Then, if OTB is not found:
+ search for releases installations: $HOME/Applications
+ OR (for Linux): /opt/otbtf > /opt/otb > /usr/local > /usr
+ OR (for Windows): C:/Program Files
Args:
prefix: prefix to search OTB in (Default value = OTB_ROOT)
scan: find otb in system known locations (Default value = True)
- scan_userdir: search for OTB release in user's home directory (Default value = True)
Returns:
otbApplication module
@@ -65,14 +70,15 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True, scan_userdir: bool = Tru
import otbApplication as otb # pylint: disable=import-outside-toplevel
return otb
- except EnvironmentError as e:
+ except SystemError as e:
raise SystemExit(f"Failed to import OTB with prefix={prefix}") from e
except ImportError as e:
__suggest_fix_import(str(e), prefix)
raise SystemExit("Failed to import OTB. Exiting.") from e
# Else try import from actual Python path
try:
- # Here, we can't properly set env variables before OTB import. We assume user did this before running python
+ # Here, we can't properly set env variables before OTB import.
+ # We assume user did this before running python
# For LD_LIBRARY_PATH problems, use OTB_ROOT instead of PYTHONPATH
import otbApplication as otb # pylint: disable=import-outside-toplevel
@@ -89,18 +95,25 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True, scan_userdir: bool = Tru
) from e
# Else search system
logger.info("Failed to import OTB. Searching for it...")
- prefix = __find_otb_root(scan_userdir)
- # Try to import one last time before raising error
+ prefix = __find_otb_root()
+ # Try auto install if shell is interactive
+ if not prefix and hasattr(sys, "ps1"):
+ if input("OTB is missing. Do you want to install it ? (y/n): ") == "y":
+ return find_otb(install_otb(*interactive_config()))
+ raise SystemError("OTB libraries not found on disk. ")
+ if not prefix:
+ raise SystemExit(
+ "OTB libraries not found on disk. "
+ "To install it, open an interactive python shell and 'import pyotb'"
+ )
+ # If OTB was found on disk, set env and try to import one last time
try:
set_environment(prefix)
import otbApplication as otb # pylint: disable=import-outside-toplevel
return otb
- except EnvironmentError as e:
+ except SystemError as e:
raise SystemExit("Auto setup for OTB env failed. Exiting.") from e
- # Unknown error
- except ModuleNotFoundError as e:
- raise SystemExit("Can't run without OTB installed. Exiting.") from e
# Help user to fix this
except ImportError as e:
__suggest_fix_import(str(e), prefix)
@@ -125,7 +138,7 @@ def set_environment(prefix: str):
# External libraries
lib_dir = __find_lib(prefix)
if not lib_dir:
- raise EnvironmentError("Can't find OTB external libraries")
+ raise SystemError("Can't find OTB external libraries")
# This does not seems to work
if sys.platform == "linux" and built_from_source:
new_ld_path = f"{lib_dir}:{os.environ.get('LD_LIBRARY_PATH') or ''}"
@@ -133,17 +146,17 @@ def set_environment(prefix: str):
# Add python bindings directory first in PYTHONPATH
otb_api = __find_python_api(lib_dir)
if not otb_api:
- raise EnvironmentError("Can't find OTB Python API")
+ raise SystemError("Can't find OTB Python API")
if otb_api not in sys.path:
sys.path.insert(0, otb_api)
- # Add /bin first in PATH, in order to avoid conflicts with another GDAL install when using os.system()
+ # Add /bin first in PATH, in order to avoid conflicts with another GDAL install
os.environ["PATH"] = f"{prefix / 'bin'}{os.pathsep}{os.environ['PATH']}"
- # Applications path (this can be tricky since OTB import will succeed even without apps)
+ # Ensure APPLICATION_PATH is set
apps_path = __find_apps_path(lib_dir)
if Path(apps_path).exists():
os.environ["OTB_APPLICATION_PATH"] = apps_path
else:
- raise EnvironmentError("Can't find OTB applications directory")
+ raise SystemError("Can't find OTB applications directory")
os.environ["LC_NUMERIC"] = "C"
os.environ["GDAL_DRIVER_PATH"] = "disable"
@@ -159,7 +172,7 @@ def set_environment(prefix: str):
gdal_data = str(prefix / "share/data")
proj_lib = str(prefix / "share/proj")
else:
- raise EnvironmentError(
+ raise SystemError(
f"Can't find GDAL location with current OTB prefix '{prefix}' or in /usr"
)
os.environ["GDAL_DATA"] = gdal_data
@@ -171,7 +184,7 @@ def __find_lib(prefix: str = None, otb_module=None):
Args:
prefix: try with OTB root directory
- otb_module: try with OTB python module (otbApplication) library path if found, else None
+ otb_module: try with otbApplication library path if found, else None
Returns:
lib path
@@ -235,12 +248,9 @@ def __find_apps_path(lib_dir: Path):
return ""
-def __find_otb_root(scan_userdir: bool = False):
+def __find_otb_root():
"""Search for OTB root directory in well known locations.
- Args:
- scan_userdir: search with glob in $HOME directory
-
Returns:
str path of the OTB directory
@@ -263,78 +273,72 @@ def __find_otb_root(scan_userdir: bool = False):
prefix = path.parent.parent.parent
else:
prefix = path.parent.parent
- prefix = prefix.absolute()
elif sys.platform == "win32":
- for path in Path("c:/Program Files").glob("**/OTB-*/lib"):
- logger.info("Found %s", path.parent)
- prefix = path.parent.absolute()
- elif sys.platform == "darwin":
- for path in (Path.home() / "Applications").glob("**/OTB-*/lib"):
- logger.info("Found %s", path.parent)
- prefix = path.parent.absolute()
- # If possible, use OTB found in user's HOME tree (this may take some time)
- if scan_userdir:
- for path in Path.home().glob("**/OTB-*/lib"):
+ for path in sorted(Path("c:/Program Files").glob("**/OTB-*/lib")):
logger.info("Found %s", path.parent)
- prefix = path.parent.absolute()
- # Return latest found prefix (and version), see precedence in function def find_otb()
- return prefix
+ prefix = path.parent
+ # Search for pyotb OTB install, or default on macOS
+ apps = Path.home() / "Applications"
+ for path in sorted(apps.glob("OTB-*/lib/")):
+ logger.info("Found %s", path.parent)
+ prefix = path.parent
+ # Return latest found prefix (and version), see precedence in find_otb() docstrings
+ if isinstance(prefix, Path):
+ return prefix.absolute()
+ return None
def __suggest_fix_import(error_message: str, prefix: str):
"""Help user to fix the OTB installation with appropriate log messages."""
logger.critical("An error occurred while importing OTB Python API")
logger.critical("OTB error message was '%s'", error_message)
- if sys.platform == "linux":
- if error_message.startswith("libpython3."):
- logger.critical(
- "It seems like you need to symlink or recompile python bindings"
- )
- if sys.executable.startswith("/usr/bin"):
- lib = (
- f"/usr/lib/x86_64-linux-gnu/libpython3.{sys.version_info.minor}.so"
- )
- if which("ctest"):
- logger.critical(
- "To recompile python bindings, use 'cd %s ; source otbenv.profile ; "
- "ctest -S share/otb/swig/build_wrapping.cmake -VV'",
- prefix,
- )
- elif Path(lib).exists():
- expect_minor = int(error_message[11])
- if expect_minor != sys.version_info.minor:
- logger.critical(
- "Python library version mismatch (OTB was expecting 3.%s) : "
- "a simple symlink may not work, depending on your python version",
- expect_minor,
- )
- target_lib = f"{prefix}/lib/libpython3.{expect_minor}.so.rh-python3{expect_minor}-1.0"
- logger.critical("Use 'ln -s %s %s'", lib, target_lib)
- else:
- logger.critical(
- "You may need to install cmake in order to recompile python bindings"
- )
- else:
- logger.critical(
- "Unable to automatically locate python dynamic library of %s",
- sys.executable,
- )
- elif sys.platform == "win32":
+ if sys.platform == "win32":
if error_message.startswith("DLL load failed"):
if sys.version_info.minor != 7:
logger.critical(
- "You need Python 3.5 (OTB releases 6.4 to 7.4) or Python 3.7 (since OTB 8)"
+ "You need Python 3.5 (OTB 6.4 to 7.4) or Python 3.7 (since OTB 8)"
)
else:
logger.critical(
"It seems that your env variables aren't properly set,"
" first use 'call otbenv.bat' then try to import pyotb once again"
)
- docs_link = "https://www.orfeo-toolbox.org/CookBook/Installation.html"
+ elif error_message.startswith("libpython3."):
+ logger.critical(
+ "It seems like you need to symlink or recompile python bindings"
+ )
+ if (
+ sys.executable.startswith("/usr/bin")
+ and which("ctest")
+ and which("python3-config")
+ ):
+ logger.critical(
+ "To compile, use 'cd %s ; source otbenv.profile ; "
+ "ctest -S share/otb/swig/build_wrapping.cmake -VV'",
+ prefix,
+ )
+ return
+ logger.critical(
+ "You may need to install cmake, python3-dev and mesa's libgl"
+ " in order to recompile python bindings"
+ )
+ expected = int(error_message[11])
+ if expected != sys.version_info.minor:
+ logger.critical(
+ "Python library version mismatch (OTB expected 3.%s) : "
+ "a symlink may not work, depending on your python version",
+ expected,
+ )
+ lib_dir = sysconfig.get_config_var("LIBDIR")
+ lib = f"{lib_dir}/libpython3.{sys.version_info.minor}.so"
+ if Path(lib).exists():
+ target = f"{prefix}/lib/libpython3.{expected}.so.1.0"
+ logger.critical("If using OTB>=8.0, try 'ln -sf %s %s'", lib, target)
logger.critical(
- "You can verify installation requirements for your OS at %s", docs_link
+ "You can verify installation requirements for your OS at %s", DOCS_URL
)
-# Since helpers is the first module to be inititialized, this will prevent pyotb to run if OTB is not found
+# This part of pyotb is the first imported during __init__ and checks if OTB is found
+# If OTB is not found, a SystemExit is raised, to prevent execution of the core module
find_otb()
diff --git a/pyotb/install.py b/pyotb/install.py
new file mode 100644
index 0000000000000000000000000000000000000000..47a6ee7252c02afa8d4f1957ba694ed20cbfcd06
--- /dev/null
+++ b/pyotb/install.py
@@ -0,0 +1,199 @@
+"""This module contains functions for interactive auto installation of OTB."""
+import json
+import os
+import re
+import subprocess
+import sys
+import sysconfig
+import tempfile
+import urllib.request
+import zipfile
+from pathlib import Path
+from shutil import which
+
+
+def interactive_config():
+ """Prompt user to configure installation variables."""
+ version = input("Choose a version to download (default is latest): ")
+ default_dir = Path.home() / "Applications"
+ path = input(f"Parent directory for installation (default is {default_dir}): ")
+ env = input("Permanently change user's environment variables ? (y/n): ") == "y"
+ return version, path, env
+
+
+def otb_latest_release_tag():
+ """Use gitlab API to find latest release tag name, but skip pre-releases."""
+ api_endpoint = "https://gitlab.orfeo-toolbox.org/api/v4/projects/53/repository/tags"
+ vers_regex = re.compile(r"^\d\.\d\.\d$") # we ignore rc-* or alpha-*
+ with urllib.request.urlopen(api_endpoint) as stream:
+ data = json.loads(stream.read())
+ releases = sorted(
+ [tag["name"] for tag in data if vers_regex.match(tag["name"])],
+ )
+ return releases[-1]
+
+
+def check_versions(sysname: str, python_minor: int, otb_major: int):
+ """Verify if python version is compatible with major OTB version.
+
+ Args:
+ sysname: OTB's system name convention (Linux64, Darwin64, Win64)
+ python_minor: minor version of python
+ otb_major: major version of OTB to be installed
+ Returns:
+ (True, 0) or (False, expected_version) if case of version conflict
+ """
+ if sysname == "Win64":
+ expected = 5 if otb_major in (6, 7) else 7
+ if python_minor == expected:
+ return True, 0
+ elif sysname == "Darwin64":
+ expected = 7, 0
+ if python_minor == expected:
+ return True, 0
+ elif sysname == "Linux64":
+ expected = 5 if otb_major in (6, 7) else 8
+ if python_minor == expected:
+ return True, 0
+ return False, expected
+
+
+def env_config_unix(otb_path: Path):
+ """Update env profile for current user with new otb_env.profile call.
+
+ Args:
+ otb_path: the path of the new OTB installation
+
+ """
+ profile = Path.home() / ".profile"
+ with profile.open("a", encoding="utf-8") as buf:
+ buf.write(f'\n. "{otb_path}/otbenv.profile"\n')
+ print(f"##### Added new environment variables to {profile}")
+
+
+def env_config_windows(otb_path: Path):
+ """Update user's registry hive with new OTB_ROOT env variable.
+
+ Args:
+ otb_path: path of the new OTB installation
+
+ """
+ import winreg # pylint: disable=import-error,import-outside-toplevel
+
+ with winreg.OpenKeyEx(
+ winreg.HKEY_CURRENT_USER, "Environment", 0, winreg.KEY_SET_VALUE
+ ) as reg_key:
+ winreg.SetValueEx(reg_key, "OTB_ROOT", 0, winreg.REG_EXPAND_SZ, str(otb_path))
+ print(
+ "##### Environment variable 'OTB_ROOT' added to user's registry. "
+ "You'll need to login / logout to apply this change."
+ )
+ reg_cmd = "reg.exe delete HKEY_CURRENT_USER\\Environment /v OTB_ROOT /f"
+ print(f"To undo this, you may use '{reg_cmd}'")
+
+
+def install_otb(version: str = "latest", path: str = "", edit_env: bool = True):
+ """Install pre-compiled OTB binaries in path, use latest release by default.
+
+ Args:
+ version: OTB version tag, e.g. '8.1.2'
+ path: installation directory, default is $HOME/Applications
+ edit_env: whether or not to permanently modify user's environment variables
+
+ Returns:
+ full path of the new installation
+
+ """
+ # Read env config
+ if sys.version_info.major == 2:
+ raise SystemExit("Python 3 is required for OTB bindings.")
+ python_minor = sys.version_info.minor
+ if not version or version == "latest":
+ version = otb_latest_release_tag()
+ name_corresp = {"linux": "Linux64", "darwin": "Darwin64", "win32": "Win64"}
+ sysname = name_corresp[sys.platform]
+ ext = "zip" if sysname == "Win64" else "run"
+ cmd = which("zsh") or which("bash") or which("sh")
+ otb_major = int(version[0])
+ check, expected = check_versions(sysname, python_minor, otb_major)
+ if sysname == "Win64" and not check:
+ raise SystemExit(
+ f"Python 3.{expected} is required to import bindings on Windows."
+ )
+ # Fetch archive and run installer
+ filename = f"OTB-{version}-{sysname}.{ext}"
+ url = f"https://www.orfeo-toolbox.org/packages/archives/OTB/{filename}"
+ tmpdir = tempfile.gettempdir()
+ tmpfile = Path(tmpdir) / filename
+ print(f"##### Downloading {url}")
+ urllib.request.urlretrieve(url, tmpfile)
+ if path:
+ default_path = False
+ path = Path(path)
+ else:
+ default_path = True
+ path = Path.home() / "Applications" / tmpfile.stem
+ if sysname == "Win64":
+ with zipfile.ZipFile(tmpfile) as zipf:
+ print("##### Extracting zip file")
+ # Unzip will always create a dir with OTB-version name
+ zipf.extractall(path.parent if default_path else path)
+ else:
+ install_cmd = f"{cmd} {tmpfile} --target {path} --accept"
+ print(f"##### Executing '{install_cmd}'\n")
+ subprocess.run(install_cmd, shell=True, check=True)
+ tmpfile.unlink() # cleaning
+
+ # Add env variable to profile
+ if edit_env:
+ if sysname == "Win64":
+ env_config_windows(path)
+ else:
+ env_config_unix(path)
+ elif not default_path:
+ ext = "bat" if sysname == "Win64" else "profile"
+ print(
+ f"Remember to call '{path}{os.sep}otbenv.{ext}' before importing pyotb, "
+ f"or add 'OTB_ROOT=\"{path}\"' to your env variables."
+ )
+ # Requirements are met, no recompilation or symlink required
+ if check:
+ return str(path)
+
+ # Else try recompile bindings : can fail because of OpenGL
+ # Here we check for /usr/bin because CMake's will find_package() only there
+ if (
+ sys.executable.startswith("/usr/bin")
+ and which("ctest")
+ and which("python3-config")
+ ):
+ try:
+ print("\n!!!!! Python version mismatch, trying to recompile bindings")
+ ctest_cmd = (
+ ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -V"
+ )
+ print(f"##### Executing '{ctest_cmd}'")
+ subprocess.run(ctest_cmd, cwd=path, check=True, shell=True)
+ return str(path)
+ except subprocess.CalledProcessError:
+ print("\nCompilation failed.")
+ # TODO: support for sudo auto build deps install using apt, pacman/yay, brew...
+ print(
+ "You need cmake, python3-dev and libgl1-mesa-dev installed."
+ "\nTrying to symlink libraries instead - this may fail with newest versions."
+ )
+
+ # Finally try with cross version python symlink (only tested on Ubuntu)
+ suffix = "so.1.0" if otb_major >= 8 else f"so.rh-python3{expected}-1.0"
+ target_lib = f"{path}/lib/libpython3.{expected}.{suffix}"
+ lib_dir = sysconfig.get_config_var("LIBDIR")
+ lib = f"{lib_dir}/libpython3.{sys.version_info.minor}.so"
+ if Path(lib).exists():
+ print(f"##### Creating symbolic link: {lib} -> {target_lib}")
+ ln_cmd = f'ln -sf "{lib}" "{target_lib}"'
+ subprocess.run(ln_cmd, executable=cmd, shell=True, check=True)
+ return str(path)
+ raise SystemError(
+ f"Unable to automatically locate library for executable '{sys.executable}', "
+ f"you could manually create a symlink from that file to {target_lib}"
+ )
diff --git a/pyproject.toml b/pyproject.toml
index 30804d40d843c4fc8d6f1a8dd017e40092fe2510..fdaa0a35167bea61244a4d4cfb500c2bbf7cd7d2 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -56,7 +56,6 @@ max-line-length = 88
max-module-lines = 2000
good-names = ["x", "y", "i", "j", "k", "e"]
disable = [
- "fixme",
"line-too-long",
"too-many-locals",
"too-many-branches",