From 8f0fe4b8dc1b406ea26108ed3a77b8ee521e533b Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@irstea.fr>
Date: Tue, 28 Jun 2022 15:33:14 +0200
Subject: [PATCH 01/22] TEST: forward/backward, with/without intermediate
outputs
---
.gitlab-ci.yml | 2 ++
1 file changed, 2 insertions(+)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a6610b0..1eecadb 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -106,3 +106,5 @@ pipeline_test:
- cd tests
- python3 pipeline_test.py
- python3 pipeline_test.py backward
+ - python3 pipeline_test.py no-intermediate-result
+ - python3 pipeline_test.py backward no-intermediate-result
--
GitLab
From 229a43c014a54ae09d269f889b43ff46aedc7f1c Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@irstea.fr>
Date: Tue, 28 Jun 2022 15:33:18 +0200
Subject: [PATCH 02/22] TEST: forward/backward, with/without intermediate
outputs
---
tests/pipeline_test.py | 17 +++++++++--------
1 file changed, 9 insertions(+), 8 deletions(-)
diff --git a/tests/pipeline_test.py b/tests/pipeline_test.py
index 3b152b5..327f110 100644
--- a/tests/pipeline_test.py
+++ b/tests/pipeline_test.py
@@ -101,15 +101,16 @@ def test_pipeline(pipeline):
pipeline: pipeline (list of pyotb objects)
"""
+ args = [arg.lower() for arg in sys.argv[1:]] if len(sys.argv) > 1 else []
report = {"shapes_errs": [], "write_errs": []}
# Test outputs shapes
- generator = enumerate(pipeline)
- if len(sys.argv) > 1:
- if "backward" in sys.argv[1].lower():
- print("Perform tests in backward mode")
- generator = enumerate(reversed(pipeline))
- for i, app in generator:
+ pipeline_items = [pipeline[-1]] if "no-intermediate-output" in args else pipeline
+ generator = lambda: enumerate(pipeline_items)
+ if "backward" in args:
+ print("Perform tests in backward mode")
+ generator = lambda: enumerate(reversed(pipeline_items))
+ for i, app in generator():
try:
print(f"Trying to access shape of app {app.name} output...")
shape = app.shape
@@ -120,7 +121,7 @@ def test_pipeline(pipeline):
report["shapes_errs"].append(i)
# Test all pipeline outputs
- for i, app in generator:
+ for i, app in generator():
if not check_app_write(app, f"/tmp/out_{i}.tif"):
report["write_errs"].append(i)
@@ -174,7 +175,7 @@ for pipeline, errs in results.items():
msg += " | "
if has_err:
nb_fails += 1
- causes = [f"{section}: " + ", ".join([str(i) for i in out_ids])
+ causes = [f"{section}: " + ", ".join([f"app{i}" for i in out_ids])
for section, out_ids in errs.items() if out_ids]
msg += "\033[91mFAIL\033[0m (" + "; ".join(causes) + ")"
else:
--
GitLab
From 919f7c918eeb1d38481915ee6bc56da8557575a3 Mon Sep 17 00:00:00 2001
From: Narcon Nicolas <nicolas.narcon@inrae.fr>
Date: Wed, 29 Jun 2022 14:12:51 +0200
Subject: [PATCH 03/22] WIP: remove execute in Operation init
---
pyotb/core.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 598185a..a3d91e8 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -802,7 +802,7 @@ class App(otbObject):
try:
self.app.Execute()
if self.__with_output():
- self.app.WriteOutput()
+ self.app.ExecuteAndWriteOutput()
self.finished = True
except (RuntimeError, FileNotFoundError) as e:
raise Exception(f'{self.name}: error during during app execution') from e
@@ -1101,7 +1101,6 @@ class Operation(otbObject):
app = App('BandMath', il=self.unique_inputs, exp=self.exp)
else:
app = App('BandMathX', il=self.unique_inputs, exp=self.exp)
- app.execute()
self.app = app.app
def create_fake_exp(self, operator, inputs, nb_bands=None):
--
GitLab
From 97ef39a2ed4f27e05bdfd6f48c5af5f3b7b045e6 Mon Sep 17 00:00:00 2001
From: Narcon Nicolas <nicolas.narcon@inrae.fr>
Date: Wed, 29 Jun 2022 14:44:46 +0200
Subject: [PATCH 04/22] WIP: loosen the condition for execution of objects in
pipelines
---
doc/index.md | 3 ++-
doc/interaction.md | 3 +++
pyotb/core.py | 2 +-
3 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/doc/index.md b/doc/index.md
index eb936d5..d73bd16 100644
--- a/doc/index.md
+++ b/doc/index.md
@@ -1,4 +1,4 @@
-# Pyotb
+# Pyotb: Orfeo Toolbox for Python
pyotb is a Python extension of Orfeo Toolbox. It has been built on top of the existing Python API of OTB, in order
to make OTB more Python friendly.
@@ -20,4 +20,5 @@ to make OTB more Python friendly.
- [Comparison between pyotb and OTB native library](comparison_otb.md)
- [OTB versions](otb_versions.md)
- [Managing loggers](managing_loggers.md)
+
## API
diff --git a/doc/interaction.md b/doc/interaction.md
index e8493a8..6c87417 100644
--- a/doc/interaction.md
+++ b/doc/interaction.md
@@ -33,6 +33,7 @@ noisy_image = inp + white_noise # magic: this is a pyotb object that has the sa
noisy_image.write('image_plus_noise.tif')
```
Limitations :
+
- The whole image is loaded into memory
- The georeference can not be modified. Thus, numpy operations can not change the image or pixel size
@@ -70,9 +71,11 @@ res = scalar_product('image1.tif', 'image2.tif') # magic: this is a pyotb objec
```
Advantages :
+
- The process supports streaming, hence the whole image is **not** loaded into memory
- Can be integrated in OTB pipelines
Limitations :
+
- It is not possible to use the tensorflow python API inside a script where OTBTF is used because of compilation issues
between Tensorflow and OTBTF, i.e. `import tensorflow` doesn't work in a script where OTBTF apps have been initialized
diff --git a/pyotb/core.py b/pyotb/core.py
index a3d91e8..f1418ca 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -176,7 +176,7 @@ class otbObject(ABC):
if isinstance(self, App):
if not self.finished:
self.execute()
- elif isinstance(self, Output):
+ elif isinstance(self, otbObject):
self.app.Execute()
# Special methods
--
GitLab
From 2ce882f1b00e034de7983fb0829089323e9fa0ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr>
Date: Wed, 29 Jun 2022 15:08:18 +0000
Subject: [PATCH 05/22] Fix test pipelines bugs
---
.gitlab-ci.yml | 19 ++++++++++++-------
tests/pipeline_test.py | 31 ++++++++++++++++---------------
2 files changed, 28 insertions(+), 22 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1eecadb..64ab7dd 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -83,6 +83,7 @@ pages:
allow_failure: false
before_script:
- pip install .
+ - cd tests
variables:
OTB_ROOT: /opt/otb
LD_LIBRARY_PATH: /opt/otb/lib
@@ -97,14 +98,18 @@ import_pyotb:
compute_ndvi:
extends: .test_base
script:
- - cd tests
- python3 ndvi_test.py
-pipeline_test:
+pipeline_tests:
extends: .test_base
script:
- - cd tests
- - python3 pipeline_test.py
- - python3 pipeline_test.py backward
- - python3 pipeline_test.py no-intermediate-result
- - python3 pipeline_test.py backward no-intermediate-result
+ - export OTB_LOGGER_LEVEL="ERROR"
+ - export PYOTB_LOGGER_LEVEL="ERROR"
+ - python3 pipeline_test.py shape
+ - python3 pipeline_test.py shape backward
+ - python3 pipeline_test.py shape no-intermediate-result
+ - python3 pipeline_test.py shape backward no-intermediate-result
+ - python3 pipeline_test.py write
+ - python3 pipeline_test.py write backward
+ - python3 pipeline_test.py write no-intermediate-result
+ - python3 pipeline_test.py write backward no-intermediate-result
diff --git a/tests/pipeline_test.py b/tests/pipeline_test.py
index 327f110..88c8212 100644
--- a/tests/pipeline_test.py
+++ b/tests/pipeline_test.py
@@ -52,7 +52,7 @@ def check_app_write(app, out):
filepath = 'Data/Input/QB_MUL_ROI_1000_100.tif'
pyotb_input = pyotb.Input(filepath)
-
+args = [arg.lower() for arg in sys.argv[1:]] if len(sys.argv) > 1 else []
def generate_pipeline(inp, building_blocks):
"""
@@ -101,7 +101,6 @@ def test_pipeline(pipeline):
pipeline: pipeline (list of pyotb objects)
"""
- args = [arg.lower() for arg in sys.argv[1:]] if len(sys.argv) > 1 else []
report = {"shapes_errs": [], "write_errs": []}
# Test outputs shapes
@@ -110,20 +109,22 @@ def test_pipeline(pipeline):
if "backward" in args:
print("Perform tests in backward mode")
generator = lambda: enumerate(reversed(pipeline_items))
- for i, app in generator():
- try:
- print(f"Trying to access shape of app {app.name} output...")
- shape = app.shape
- print(f"App {app.name} output shape is {shape}")
- except Exception as e:
- print("\n\033[91mGET SHAPE ERROR\033[0m")
- print(e)
- report["shapes_errs"].append(i)
+ if "shape" in args:
+ for i, app in generator():
+ try:
+ print(f"Trying to access shape of app {app.name} output...")
+ shape = app.shape
+ print(f"App {app.name} output shape is {shape}")
+ except Exception as e:
+ print("\n\033[91mGET SHAPE ERROR\033[0m")
+ print(e)
+ report["shapes_errs"].append(i)
# Test all pipeline outputs
- for i, app in generator():
- if not check_app_write(app, f"/tmp/out_{i}.tif"):
- report["write_errs"].append(i)
+ if "write" in args:
+ for i, app in generator():
+ if not check_app_write(app, f"/tmp/out_{i}.tif"):
+ report["write_errs"].append(i)
return report
@@ -162,7 +163,7 @@ for pipeline in pipelines:
# Summary
cols = max([len(pipeline2str(pipeline)) for pipeline in pipelines]) + 1
-print("Tests summary:")
+print(f'Tests summary (\033[93mTest options: {"; ".join(args)}\033[0m)')
print("Pipeline".ljust(cols) + " | Status (reason)")
print("-" * cols + "-|-" + "-" * 20)
nb_fails = 0
--
GitLab
From e93d1f4a0fc28a338085832a1117e314fcf9afdd Mon Sep 17 00:00:00 2001
From: Narcon Nicolas <nicolas.narcon@inrae.fr>
Date: Wed, 29 Jun 2022 17:10:09 +0200
Subject: [PATCH 06/22] WIP: removing all PropagateConnectMode(False)
---
pyotb/core.py | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index f1418ca..c83677c 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -120,8 +120,6 @@ class otbObject(ABC):
if key in dtypes:
self.app.SetParameterOutputImagePixelType(key, dtypes[key])
- self.app.PropagateConnectMode(False)
-
if isinstance(self, App):
return self.execute()
@@ -781,7 +779,8 @@ class App(otbObject):
self.app.SetParameterOutputImagePixelType(key, typ)
# Here we make sure that intermediate outputs will be flushed to disk
if self.__with_output():
- self.app.PropagateConnectMode(False)
+ pass
+ # self.app.PropagateConnectMode(True)
# Run app, write output if needed, update `finished` property
if execute or not self.output_param:
self.execute()
--
GitLab
From ede24c33b0a752d382b069148fdc4e2f7f93227e Mon Sep 17 00:00:00 2001
From: Narcon Nicolas <nicolas.narcon@inrae.fr>
Date: Wed, 29 Jun 2022 17:36:29 +0200
Subject: [PATCH 07/22] WIP: keep reference to pyotb app as attribute for
Slicer and Operation
---
pyotb/core.py | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index c83677c..2003cae 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -175,7 +175,8 @@ class otbObject(ABC):
if not self.finished:
self.execute()
elif isinstance(self, otbObject):
- self.app.Execute()
+ if not self.pyotb_app.finished:
+ self.pyotb_app.execute()
# Special methods
def __getitem__(self, key):
@@ -615,8 +616,8 @@ class Slicer(otbObject):
# Execute app
app.set_parameters(**parameters)
app.execute()
- # Keeping the OTB app, not the pyotb app
- self.app = app.app
+ # Keeping the OTB app and the pyotb app
+ self.pyotb_app, self.app = app, app.app
# These are some attributes when the user simply wants to extract *one* band to be used in an Operation
if not spatial_slicing and isinstance(channels, list) and len(channels) == 1:
@@ -1100,7 +1101,9 @@ class Operation(otbObject):
app = App('BandMath', il=self.unique_inputs, exp=self.exp)
else:
app = App('BandMathX', il=self.unique_inputs, exp=self.exp)
- self.app = app.app
+
+ # Keeping the OTB app and the pyotb app
+ self.pyotb_app, self.app = app, app.app
def create_fake_exp(self, operator, inputs, nb_bands=None):
"""
--
GitLab
From a47be77df3d06d6789053d42d583f5a983ab9dca Mon Sep 17 00:00:00 2001
From: Narcon Nicolas <nicolas.narcon@inrae.fr>
Date: Wed, 29 Jun 2022 17:45:32 +0200
Subject: [PATCH 08/22] WIP: keep reference to pyotb app as attribute for Input
and Output
---
pyotb/core.py | 13 ++++++++-----
1 file changed, 8 insertions(+), 5 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 2003cae..7d198b4 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -640,7 +640,9 @@ class Input(otbObject):
self.filepath = filepath
self.name = f'Input from {filepath}'
app = App('ExtractROI', filepath, execute=True, propagate_pixel_type=True)
- self.app = app.app
+
+ # Keeping the OTB app and the pyotb app
+ self.pyotb_app, self.app = app, app.app
def __str__(self):
"""
@@ -656,14 +658,15 @@ class Output(otbObject):
Class for output of an app
"""
- def __init__(self, app, output_parameter_key):
+ def __init__(self, pyotb_app, output_parameter_key):
"""
Args:
- app: The OTB application
+ app: The pyotb App
output_parameter_key: Output parameter key
"""
- self.app = app # keeping a reference of the OTB app
+ # Keeping the OTB app and the pyotb app
+ self.pyotb_app, self.app = app, app.app
self.output_parameter_key = output_parameter_key
self.name = f'Output {output_parameter_key} from {self.app.GetName()}'
@@ -992,7 +995,7 @@ class App(otbObject):
"""
for key in self.app.GetParametersKeys():
if key in self.output_parameters_keys: # raster outputs
- output = Output(self.app, key)
+ output = Output(self, key)
setattr(self, key, output)
elif key not in ('parameters',): # any other attributes (scalars...)
try:
--
GitLab
From 1e5608336b4ff291a12cb8124c05b9853280393e Mon Sep 17 00:00:00 2001
From: Narcon Nicolas <nicolas.narcon@inrae.fr>
Date: Wed, 29 Jun 2022 17:52:42 +0200
Subject: [PATCH 09/22] FIX: typo
---
pyotb/core.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 7d198b4..9e6926f 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -666,7 +666,7 @@ class Output(otbObject):
"""
# Keeping the OTB app and the pyotb app
- self.pyotb_app, self.app = app, app.app
+ self.pyotb_app, self.app = pyotb_app, pyotb_app.app
self.output_parameter_key = output_parameter_key
self.name = f'Output {output_parameter_key} from {self.app.GetName()}'
--
GitLab
From 3cb32c65ff933a2341ef390dd2f72a50c7cc4b56 Mon Sep 17 00:00:00 2001
From: Narcon Nicolas <nicolas.narcon@inrae.fr>
Date: Wed, 29 Jun 2022 18:09:51 +0200
Subject: [PATCH 10/22] FIX: remove execute in Slicer
---
pyotb/core.py | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 9e6926f..f35a465 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -570,8 +570,6 @@ class Slicer(otbObject):
# Trigger source app execution if needed
x.execute_if_needed()
app = App('ExtractROI', {'in': x, 'mode': 'extent'}, propagate_pixel_type=True)
- # First exec required in order to read image dim
- app.app.Execute()
parameters = {}
# Channel slicing
@@ -615,7 +613,7 @@ class Slicer(otbObject):
spatial_slicing = True
# Execute app
app.set_parameters(**parameters)
- app.execute()
+
# Keeping the OTB app and the pyotb app
self.pyotb_app, self.app = app, app.app
--
GitLab
From 34f4a4499fc61a208c91ac131f22598dcea2069e Mon Sep 17 00:00:00 2001
From: Narcon Nicolas <nicolas.narcon@inrae.fr>
Date: Thu, 30 Jun 2022 09:08:29 +0200
Subject: [PATCH 11/22] FIX: add execute in Slicer
---
pyotb/core.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/pyotb/core.py b/pyotb/core.py
index f35a465..97e91e8 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -570,6 +570,7 @@ class Slicer(otbObject):
# Trigger source app execution if needed
x.execute_if_needed()
app = App('ExtractROI', {'in': x, 'mode': 'extent'}, propagate_pixel_type=True)
+ app.app.Execute()
parameters = {}
# Channel slicing
--
GitLab
From 47b1647f1deea28d5868d7417db8deeba67fa0c2 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 30 Jun 2022 11:20:22 +0200
Subject: [PATCH 12/22] ENH: separate tests in different jobs
---
.gitlab-ci.yml | 33 ++++++++++++++++++++++++++++++---
1 file changed, 30 insertions(+), 3 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 64ab7dd..b7d75da 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -100,16 +100,43 @@ compute_ndvi:
script:
- python3 ndvi_test.py
-pipeline_tests:
+pipeline_test_shape:
extends: .test_base
script:
- - export OTB_LOGGER_LEVEL="ERROR"
- - export PYOTB_LOGGER_LEVEL="ERROR"
- python3 pipeline_test.py shape
+
+pipeline_test_shape_backward:
+ extends: .test_base
+ script:
- python3 pipeline_test.py shape backward
+
+pipeline_test_shape_nointermediate:
+ extends: .test_base
+ script:
- python3 pipeline_test.py shape no-intermediate-result
+
+pipeline_test_shape_backward_nointermediate:
+ extends: .test_base
+ script:
- python3 pipeline_test.py shape backward no-intermediate-result
+
+pipeline_test_write:
+ extends: .test_base
+ script:
- python3 pipeline_test.py write
+
+pipeline_test_write_backward:
+ extends: .test_base
+ script:
- python3 pipeline_test.py write backward
+
+pipeline_test_write_nointermediate:
+ extends: .test_base
+ script:
- python3 pipeline_test.py write no-intermediate-result
+
+pipeline_test_write_backward_nointermediate:
+ extends: .test_base
+ script:
- python3 pipeline_test.py write backward no-intermediate-result
+
--
GitLab
From b9abb554b599cdc02c37e87a9d73b315c2e52442 Mon Sep 17 00:00:00 2001
From: Narcon Nicolas <nicolas.narcon@inrae.fr>
Date: Thu, 30 Jun 2022 16:47:02 +0200
Subject: [PATCH 13/22] FIX: in Slicer, execute app only when slicing channels
---
pyotb/core.py | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 97e91e8..749e364 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -567,15 +567,14 @@ class Slicer(otbObject):
# Initialize the app that will be used for writing the slicer
self.output_parameter_key = 'out'
self.name = 'Slicer'
- # Trigger source app execution if needed
- x.execute_if_needed()
app = App('ExtractROI', {'in': x, 'mode': 'extent'}, propagate_pixel_type=True)
- app.app.Execute()
parameters = {}
# Channel slicing
- nb_channels = get_nbchannels(x)
if channels != slice(None, None, None):
+ # Trigger source app execution if needed
+ nb_channels = get_nbchannels(x)
+ app.app.Execute() # this is needed by ExtractROI for setting the `cl` parameter
# if needed, converting int to list
if isinstance(channels, int):
channels = [channels]
--
GitLab
From de5ee0143cca2987c04ce667ca6a6021c2f30b76 Mon Sep 17 00:00:00 2001
From: Narcon Nicolas <nicolas.narcon@inrae.fr>
Date: Thu, 30 Jun 2022 16:49:22 +0200
Subject: [PATCH 14/22] DOC: add troubleshooting section + change site name
---
doc/index.md | 2 ++
doc/troubleshooting.md | 74 ++++++++++++++++++++++++++++++++++++++++++
mkdocs.yml | 5 +--
3 files changed, 79 insertions(+), 2 deletions(-)
create mode 100644 doc/troubleshooting.md
diff --git a/doc/index.md b/doc/index.md
index d73bd16..efebb29 100644
--- a/doc/index.md
+++ b/doc/index.md
@@ -20,5 +20,7 @@ to make OTB more Python friendly.
- [Comparison between pyotb and OTB native library](comparison_otb.md)
- [OTB versions](otb_versions.md)
- [Managing loggers](managing_loggers.md)
+- [Troubleshooting & limitations](troubleshooting.md)
+
## API
diff --git a/doc/troubleshooting.md b/doc/troubleshooting.md
new file mode 100644
index 0000000..f76da8e
--- /dev/null
+++ b/doc/troubleshooting.md
@@ -0,0 +1,74 @@
+## Troubleshooting: Known limitations
+
+### Failure of intermediate writing
+
+When chaining applications in-memory, there may be some problems when writing intermediate results, depending on the order
+the writings are requested. Some examples can be found below:
+
+#### Example of failures involving slicing
+
+For some applications (non-exhaustive know list: OpticalCalibration, DynamicConvert, BandMath), we can face unexpected
+failures when using channels slicing
+```python
+import pyotb
+
+inp = pyotb.DynamicConvert('raster.tif')
+one_band = inp[:, :, 1]
+
+# this works
+one_band.write('one_band.tif')
+
+# this works
+one_band.write('one_band.tif')
+inp.write('stretched.tif')
+
+# this does not work
+inp.write('stretched.tif')
+one_band.write('one_band.tif') # Failure here
+```
+
+However, when using only spatial slicing, no issue has been reported:
+
+```python
+import pyotb
+
+inp = pyotb.DynamicConvert('raster.tif')
+one_band = inp[:100, :100, :]
+
+# this works
+inp.write('stretched.tif')
+one_band.write('one_band.tif')
+```
+
+
+#### Example of failures involving arithmetic operation
+
+One can meet errors when using arithmetic operations on monoband raster. This limitation is related to BandMath.
+
+```python
+import pyotb
+
+inp = pyotb.Input('1band_raster.tif')
+absolute = abs(inp)
+one_band = absolute[:, :, 0]
+
+# this does not work
+absolute.write('absolute.tif')
+one_band.write('one_band.tif') # Failure here
+
+```
+
+However, the same code with a multibands raster doesn't raise any error
+
+```python
+import pyotb
+
+inp = pyotb.Input('4bands_raster.tif')
+absolute = abs(inp)
+one_band = absolute[:, :, 0]
+
+# this works
+one_band.write('one_band.tif')
+absolute.write('absolute.tif')
+
+```
\ No newline at end of file
diff --git a/mkdocs.yml b/mkdocs.yml
index 7dcadf5..07c04c2 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -27,7 +27,7 @@ plugins:
nav:
- Home: index.md
- Get Started:
- - Installation: installation.md
+ - Installation of pyotb: installation.md
- How to use pyotb: quickstart.md
- Useful features: features.md
- Functions: functions.md
@@ -39,6 +39,7 @@ nav:
- Comparison between pyotb and OTB native library: comparison_otb.md
- OTB versions: otb_versions.md
- Managing loggers: managing_loggers.md
+ - Troubleshooting & limitations: troubleshooting.md
- API:
- pyotb:
- core: reference/pyotb/core.md
@@ -69,7 +70,7 @@ markdown_extensions:
- pymdownx.snippets
# rest of the navigation..
-site_name: pyotb
+site_name: "pyotb documentation: a Python extension of OTB"
repo_url: https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb
repo_name: pyotb
docs_dir: doc/
--
GitLab
From e918b941f8c8d41afe3c698adeacf100b80cda39 Mon Sep 17 00:00:00 2001
From: Narcon Nicolas <nicolas.narcon@inrae.fr>
Date: Fri, 1 Jul 2022 11:17:02 +0200
Subject: [PATCH 15/22] DOC: more troubleshooting
---
doc/troubleshooting.md | 41 +++++++++++++++++++----------------------
1 file changed, 19 insertions(+), 22 deletions(-)
diff --git a/doc/troubleshooting.md b/doc/troubleshooting.md
index f76da8e..aae8be6 100644
--- a/doc/troubleshooting.md
+++ b/doc/troubleshooting.md
@@ -27,7 +27,18 @@ inp.write('stretched.tif')
one_band.write('one_band.tif') # Failure here
```
-However, when using only spatial slicing, no issue has been reported:
+When writing is triggered right after the application declaration, no problem occurs:
+```python
+import pyotb
+
+inp = pyotb.DynamicConvert('raster.tif')
+inp.write('stretched.tif')
+
+one_band = inp[:, :, 1]
+one_band.write('one_band.tif') # no failure
+```
+
+Also, when using only spatial slicing, no issue has been reported:
```python
import pyotb
@@ -43,32 +54,18 @@ one_band.write('one_band.tif')
#### Example of failures involving arithmetic operation
-One can meet errors when using arithmetic operations on monoband raster. This limitation is related to BandMath.
+One can meet errors when using arithmetic operations raster at the end of a pipeline when DynamicConvert, BandMath or
+OpticalCalibration is involved:
```python
import pyotb
-inp = pyotb.Input('1band_raster.tif')
+inp = pyotb.DynamicConvert('raster.tif')
+inp_new = pyotb.ManageNoData(inp, mode='changevalue')
absolute = abs(inp)
-one_band = absolute[:, :, 0]
# this does not work
-absolute.write('absolute.tif')
-one_band.write('one_band.tif') # Failure here
-
+inp.write('one_band.tif')
+inp_new.write('one_band_nodata.tif') # Failure here
+absolute.write('absolute.tif') # Failure here
```
-
-However, the same code with a multibands raster doesn't raise any error
-
-```python
-import pyotb
-
-inp = pyotb.Input('4bands_raster.tif')
-absolute = abs(inp)
-one_band = absolute[:, :, 0]
-
-# this works
-one_band.write('one_band.tif')
-absolute.write('absolute.tif')
-
-```
\ No newline at end of file
--
GitLab
From 35c4d6dbeda3178df09d5a0d5be87c145f19563a Mon Sep 17 00:00:00 2001
From: Narcon Nicolas <nicolas.narcon@inrae.fr>
Date: Fri, 1 Jul 2022 11:18:17 +0200
Subject: [PATCH 16/22] REFAC: reorder classes
---
pyotb/core.py | 460 +++++++++++++++++++++++++-------------------------
1 file changed, 226 insertions(+), 234 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 749e364..b13c7d4 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -548,185 +548,12 @@ class otbObject(ABC):
return NotImplemented
-class Slicer(otbObject):
- """Slicer objects i.e. when we call something like raster[:, :, 2] from Python"""
-
- def __init__(self, x, rows, cols, channels):
- """
- Create a slicer object, that can be used directly for writing or inside a BandMath :
- - an ExtractROI app that handles extracting bands and ROI and can be written to disk or used in pipelines
- - in case the user only wants to extract one band, an expression such as "im1b#"
-
- Args:
- x: input
- rows: rows slicing (e.g. 100:2000)
- cols: columns slicing (e.g. 100:2000)
- channels: channels, can be slicing, list or int
-
- """
- # Initialize the app that will be used for writing the slicer
- self.output_parameter_key = 'out'
- self.name = 'Slicer'
- app = App('ExtractROI', {'in': x, 'mode': 'extent'}, propagate_pixel_type=True)
-
- parameters = {}
- # Channel slicing
- if channels != slice(None, None, None):
- # Trigger source app execution if needed
- nb_channels = get_nbchannels(x)
- app.app.Execute() # this is needed by ExtractROI for setting the `cl` parameter
- # if needed, converting int to list
- if isinstance(channels, int):
- channels = [channels]
- # if needed, converting slice to list
- elif isinstance(channels, slice):
- channels_start = channels.start if channels.start is not None else 0
- channels_end = channels.stop if channels.stop is not None else nb_channels
- channels_step = channels.step if channels.step is not None else 1
- channels = range(channels_start, channels_end, channels_step)
- elif isinstance(channels, tuple):
- channels = list(channels)
- elif not isinstance(channels, list):
- raise ValueError(f'Invalid type for channels, should be int, slice or list of bands. : {channels}')
-
- # Change the potential negative index values to reverse index
- channels = [c if c >= 0 else nb_channels + c for c in channels]
- parameters.update({'cl': [f'Channel{i + 1}' for i in channels]})
-
- # Spatial slicing
- spatial_slicing = False
- # TODO: handle PixelValue app so that accessing value is possible, e.g. raster[120, 200, 0]
- # TODO TBD: handle the step value in the slice so that NN undersampling is possible ? e.g. obj[::2, ::2]
- if rows.start is not None:
- parameters.update({'mode.extent.uly': rows.start})
- spatial_slicing = True
- if rows.stop is not None and rows.stop != -1:
- parameters.update(
- {'mode.extent.lry': rows.stop - 1}) # subtract 1 to be compliant with python convention
- spatial_slicing = True
- if cols.start is not None:
- parameters.update({'mode.extent.ulx': cols.start})
- spatial_slicing = True
- if cols.stop is not None and cols.stop != -1:
- parameters.update(
- {'mode.extent.lrx': cols.stop - 1}) # subtract 1 to be compliant with python convention
- spatial_slicing = True
- # Execute app
- app.set_parameters(**parameters)
-
- # Keeping the OTB app and the pyotb app
- self.pyotb_app, self.app = app, app.app
-
- # These are some attributes when the user simply wants to extract *one* band to be used in an Operation
- if not spatial_slicing and isinstance(channels, list) and len(channels) == 1:
- self.one_band_sliced = channels[0] + 1 # OTB convention: channels start at 1
- self.input = x
-
-
-class Input(otbObject):
- """
- Class for transforming a filepath to pyOTB object
- """
-
- def __init__(self, filepath):
- """
- Args:
- filepath: raster file path
-
- """
- self.output_parameter_key = 'out'
- self.filepath = filepath
- self.name = f'Input from {filepath}'
- app = App('ExtractROI', filepath, execute=True, propagate_pixel_type=True)
-
- # Keeping the OTB app and the pyotb app
- self.pyotb_app, self.app = app, app.app
-
- def __str__(self):
- """
- Returns:
- string representation
-
- """
- return f'<pyotb.Input object from {self.filepath}>'
-
-
-class Output(otbObject):
- """
- Class for output of an app
- """
-
- def __init__(self, pyotb_app, output_parameter_key):
- """
- Args:
- app: The pyotb App
- output_parameter_key: Output parameter key
-
- """
- # Keeping the OTB app and the pyotb app
- self.pyotb_app, self.app = pyotb_app, pyotb_app.app
- self.output_parameter_key = output_parameter_key
- self.name = f'Output {output_parameter_key} from {self.app.GetName()}'
-
- def __str__(self):
- """
- Returns:
- string representation
-
- """
- return f'<pyotb.Output {self.app.GetName()} object, id {id(self)}>'
-
-
class App(otbObject):
"""
Class of an OTB app
"""
_name = ""
- @property
- def name(self):
- """
- Returns:
- user's defined name or appname
-
- """
- return self._name or self.appname
-
- @name.setter
- def name(self, val):
- """Set custom App name
-
- Args:
- val: new name
-
- """
- self._name = val
-
- @property
- def finished(self):
- """
- Property to store whether App has been executed but False if any output file is missing
-
- Returns:
- True if exec ended and output files are found else False
-
- """
- if self._ended and self.find_output():
- return True
- return False
-
- @finished.setter
- def finished(self, val):
- """
- Value `_ended` will be set to True right after App.execute() or App.write(),
- then find_output() is called when accessing the property
-
- Args:
- val: True if execution ended without exceptions
-
- """
- self._ended = val
-
def __init__(self, appname, *args, execute=False, image_dic=None, otb_stdout=True,
pixel_type=None, propagate_pixel_type=False, **kwargs):
"""
@@ -790,6 +617,54 @@ class App(otbObject):
else:
self.__save_objects()
+ def get_output_parameters_keys(self):
+ """Get raster output parameter keys
+
+ Returns:
+ output parameters keys
+ """
+ return [param for param in self.app.GetParametersKeys()
+ if self.app.GetParameterType(param) == otb.ParameterType_OutputImage]
+
+ def set_parameters(self, *args, **kwargs):
+ """Set some parameters of the app. When useful, e.g. for images list, this function appends the parameters
+ instead of overwriting them. Handles any parameters, i.e. in-memory & filepaths
+
+ Args:
+ *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved
+ (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit")
+ - string, App or Output, useful when the user implicitly wants to set the param "in"
+ - list, useful when the user implicitly wants to set the param "il"
+ **kwargs: keyword arguments e.g. il=['input1.tif', oApp_object2, App_object3.out], out='output.tif'
+
+ Raises:
+ Exception: when the setting of a parameter failed
+
+ """
+ parameters = kwargs
+ parameters.update(self.__parse_args(args))
+ # Going through all arguments
+ for param, obj in parameters.items():
+ if param not in self.app.GetParametersKeys():
+ raise Exception(f"{self.name}: parameter '{param}' was not recognized. "
+ f"Available keys are {self.app.GetParametersKeys()}")
+ # When the parameter expects a list, if needed, change the value to list
+ if self.__is_key_list(param) and not isinstance(obj, (list, tuple)):
+ parameters[param] = [obj]
+ obj = [obj]
+ logger.warning('%s: Argument for parameter "%s" was converted to list', self.name, param)
+ try:
+ # This is when we actually call self.app.SetParameter*
+ self.__set_param(param, obj)
+ except (RuntimeError, TypeError, ValueError, KeyError) as e:
+ raise Exception(f"{self.name}: something went wrong before execution "
+ f"(while setting parameter {param} to '{obj}')") from e
+
+ # Update App's parameters attribute
+ self.parameters.update(parameters)
+ if self.preserve_dtype:
+ self.__propagate_pixel_type()
+
def execute(self):
"""
Execute with appropriate and outputs to disk if any output parameter was set
@@ -798,7 +673,6 @@ class App(otbObject):
boolean flag that indicate if command executed with success
"""
- success = False
logger.debug("%s: run execute() with parameters=%s", self.name, self.parameters)
try:
self.app.Execute()
@@ -814,6 +688,50 @@ class App(otbObject):
return success
+ @property
+ def name(self):
+ """
+ Returns:
+ user's defined name or appname
+
+ """
+ return self._name or self.appname
+
+ @name.setter
+ def name(self, val):
+ """Set custom App name
+
+ Args:
+ val: new name
+
+ """
+ self._name = val
+
+ @property
+ def finished(self):
+ """
+ Property to store whether App has been executed but False if any output file is missing
+
+ Returns:
+ True if exec ended and output files are found else False
+
+ """
+ if self._ended and self.find_output():
+ return True
+ return False
+
+ @finished.setter
+ def finished(self, val):
+ """
+ Value `_ended` will be set to True right after App.execute() or App.write(),
+ then find_output() is called when accessing the property
+
+ Args:
+ val: True if execution ended without exceptions
+
+ """
+ self._ended = val
+
def find_output(self):
"""
Find output files on disk using parameters
@@ -865,54 +783,6 @@ class App(otbObject):
if memory:
self.app.FreeRessources()
- def get_output_parameters_keys(self):
- """Get raster output parameter keys
-
- Returns:
- output parameters keys
- """
- return [param for param in self.app.GetParametersKeys()
- if self.app.GetParameterType(param) == otb.ParameterType_OutputImage]
-
- def set_parameters(self, *args, **kwargs):
- """Set some parameters of the app. When useful, e.g. for images list, this function appends the parameters
- instead of overwriting them. Handles any parameters, i.e. in-memory & filepaths
-
- Args:
- *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved
- (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit")
- - string, App or Output, useful when the user implicitly wants to set the param "in"
- - list, useful when the user implicitly wants to set the param "il"
- **kwargs: keyword arguments e.g. il=['input1.tif', oApp_object2, App_object3.out], out='output.tif'
-
- Raises:
- Exception: when the setting of a parameter failed
-
- """
- parameters = kwargs
- parameters.update(self.__parse_args(args))
- # Going through all arguments
- for param, obj in parameters.items():
- if param not in self.app.GetParametersKeys():
- raise Exception(f"{self.name}: parameter '{param}' was not recognized. "
- f"Available keys are {self.app.GetParametersKeys()}")
- # When the parameter expects a list, if needed, change the value to list
- if self.__is_key_list(param) and not isinstance(obj, (list, tuple)):
- parameters[param] = [obj]
- obj = [obj]
- logger.warning('%s: Argument for parameter "%s" was converted to list', self.name, param)
- try:
- # This is when we actually call self.app.SetParameter*
- self.__set_param(param, obj)
- except (RuntimeError, TypeError, ValueError, KeyError) as e:
- raise Exception(f"{self.name}: something went wrong before execution "
- f"(while setting parameter {param} to '{obj}')") from e
-
- # Update App's parameters attribute
- self.parameters.update(parameters)
- if self.preserve_dtype:
- self.__propagate_pixel_type()
-
# Private functions
@staticmethod
def __parse_args(args):
@@ -1005,22 +875,15 @@ class App(otbObject):
"""
Check if a key of the App is an input parameter list
"""
- return self.app.GetParameterType(key) in (
- otb.ParameterType_InputImageList,
- otb.ParameterType_StringList,
- otb.ParameterType_InputFilenameList,
- otb.ParameterType_InputVectorDataList,
- otb.ParameterType_ListView
- )
+ return self.app.GetParameterType(key) in (otb.ParameterType_InputImageList, otb.ParameterType_StringList,
+ otb.ParameterType_InputFilenameList, otb.ParameterType_ListView,
+ otb.ParameterType_InputVectorDataList)
def __is_key_images_list(self, key):
"""
Check if a key of the App is an input parameter image list
"""
- return self.app.GetParameterType(key) in (
- otb.ParameterType_InputImageList,
- otb.ParameterType_InputFilenameList
- )
+ return self.app.GetParameterType(key) in (otb.ParameterType_InputImageList, otb.ParameterType_InputFilenameList)
# Special methods
def __str__(self):
@@ -1030,6 +893,135 @@ class App(otbObject):
return f'<pyotb.App {self.appname} object id {id(self)}>'
+class Slicer(otbObject):
+ """Slicer objects i.e. when we call something like raster[:, :, 2] from Python"""
+
+ def __init__(self, x, rows, cols, channels):
+ """
+ Create a slicer object, that can be used directly for writing or inside a BandMath. It contains :
+ - an ExtractROI app that handles extracting bands and ROI and can be written to disk or used in pipelines
+ - in case the user only wants to extract one band, an expression such as "im1b#"
+
+ Args:
+ x: input
+ rows: rows slicing (e.g. 100:2000)
+ cols: columns slicing (e.g. 100:2000)
+ channels: channels, can be slicing, list or int
+
+ """
+ # Initialize the app that will be used for writing the slicer
+ self.output_parameter_key = 'out'
+ self.name = 'Slicer'
+ app = App('ExtractROI', {'in': x, 'mode': 'extent'}, propagate_pixel_type=True)
+
+ parameters = {}
+ # Channel slicing
+ if channels != slice(None, None, None):
+ # Trigger source app execution if needed
+ nb_channels = get_nbchannels(x)
+ app.app.Execute() # this is needed by ExtractROI for setting the `cl` parameter
+ # if needed, converting int to list
+ if isinstance(channels, int):
+ channels = [channels]
+ # if needed, converting slice to list
+ elif isinstance(channels, slice):
+ channels_start = channels.start if channels.start is not None else 0
+ channels_end = channels.stop if channels.stop is not None else nb_channels
+ channels_step = channels.step if channels.step is not None else 1
+ channels = range(channels_start, channels_end, channels_step)
+ elif isinstance(channels, tuple):
+ channels = list(channels)
+ elif not isinstance(channels, list):
+ raise ValueError(f'Invalid type for channels, should be int, slice or list of bands. : {channels}')
+
+ # Change the potential negative index values to reverse index
+ channels = [c if c >= 0 else nb_channels + c for c in channels]
+ parameters.update({'cl': [f'Channel{i + 1}' for i in channels]})
+
+ # Spatial slicing
+ spatial_slicing = False
+ # TODO: handle PixelValue app so that accessing value is possible, e.g. raster[120, 200, 0]
+ # TODO TBD: handle the step value in the slice so that NN undersampling is possible ? e.g. raster[::2, ::2]
+ if rows.start is not None:
+ parameters.update({'mode.extent.uly': rows.start})
+ spatial_slicing = True
+ if rows.stop is not None and rows.stop != -1:
+ parameters.update(
+ {'mode.extent.lry': rows.stop - 1}) # subtract 1 to be compliant with python convention
+ spatial_slicing = True
+ if cols.start is not None:
+ parameters.update({'mode.extent.ulx': cols.start})
+ spatial_slicing = True
+ if cols.stop is not None and cols.stop != -1:
+ parameters.update(
+ {'mode.extent.lrx': cols.stop - 1}) # subtract 1 to be compliant with python convention
+ spatial_slicing = True
+ # Execute app
+ app.set_parameters(**parameters)
+
+ # Keeping the OTB app and the pyotb app
+ self.pyotb_app, self.app = app, app.app
+
+ # These are some attributes when the user simply wants to extract *one* band to be used in an Operation
+ if not spatial_slicing and isinstance(channels, list) and len(channels) == 1:
+ self.one_band_sliced = channels[0] + 1 # OTB convention: channels start at 1
+ self.input = x
+
+
+class Input(otbObject):
+ """
+ Class for transforming a filepath to pyOTB object
+ """
+
+ def __init__(self, filepath):
+ """
+ Args:
+ filepath: raster file path
+
+ """
+ self.output_parameter_key = 'out'
+ self.filepath = filepath
+ self.name = f'Input from {filepath}'
+ app = App('ExtractROI', filepath, execute=True, propagate_pixel_type=True)
+
+ # Keeping the OTB app and the pyotb app
+ self.pyotb_app, self.app = app, app.app
+
+ def __str__(self):
+ """
+ Returns:
+ string representation
+
+ """
+ return f'<pyotb.Input object from {self.filepath}>'
+
+
+class Output(otbObject):
+ """
+ Class for output of an app
+ """
+
+ def __init__(self, pyotb_app, output_parameter_key):
+ """
+ Args:
+ app: The pyotb App
+ output_parameter_key: Output parameter key
+
+ """
+ # Keeping the OTB app and the pyotb app
+ self.pyotb_app, self.app = pyotb_app, pyotb_app.app
+ self.output_parameter_key = output_parameter_key
+ self.name = f'Output {output_parameter_key} from {self.app.GetName()}'
+
+ def __str__(self):
+ """
+ Returns:
+ string representation
+
+ """
+ return f'<pyotb.Output {self.app.GetName()} object, id {id(self)}>'
+
+
class Operation(otbObject):
"""
Class for arithmetic/math operations done in Python.
@@ -1056,7 +1048,7 @@ class Operation(otbObject):
"""
Given some inputs and an operator, this function enables to transform this into an OTB application.
Operations generally involve 2 inputs (+, -...). It can have only 1 input for `abs` operator.
- It can have 3 inputs for the ternary operator `cond ? x : y`,
+ It can have 3 inputs for the ternary operator `cond ? x : y`.
Args:
operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ?
--
GitLab
From 9c3d4fad8f569abfb601febf7a889894afd4158d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20Nar=C3=A7on?= <nicolas.narcon@inrae.fr>
Date: Mon, 4 Jul 2022 13:14:38 +0200
Subject: [PATCH 17/22] REFAC: revert contructor names to the initial state for
backward comp
---
pyotb/core.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index b13c7d4..fc45d14 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1001,7 +1001,7 @@ class Output(otbObject):
Class for output of an app
"""
- def __init__(self, pyotb_app, output_parameter_key):
+ def __init__(self, app, output_parameter_key):
"""
Args:
app: The pyotb App
@@ -1009,7 +1009,7 @@ class Output(otbObject):
"""
# Keeping the OTB app and the pyotb app
- self.pyotb_app, self.app = pyotb_app, pyotb_app.app
+ self.pyotb_app, self.app = app, app.app
self.output_parameter_key = output_parameter_key
self.name = f'Output {output_parameter_key} from {self.app.GetName()}'
--
GitLab
From a8cd4e07a959aad437e94b543540fb1e313b92a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20Nar=C3=A7on?= <nicolas.narcon@inrae.fr>
Date: Mon, 4 Jul 2022 13:22:53 +0200
Subject: [PATCH 18/22] FIX: do not set param when value is None
---
pyotb/core.py | 55 ++++++++++++++++++++++++++-------------------------
1 file changed, 28 insertions(+), 27 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index fc45d14..5ef8c6c 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -803,33 +803,34 @@ class App(otbObject):
"""
Set one parameter, decide which otb.Application method to use depending on target object
"""
- # Single-parameter cases
- if isinstance(obj, otbObject):
- self.app.ConnectImage(param, obj.app, obj.output_param)
- elif isinstance(obj, otb.Application): # this is for backward comp with plain OTB
- outparamkey = [param for param in obj.GetParametersKeys()
- if obj.GetParameterType(param) == otb.ParameterType_OutputImage][0]
- self.app.ConnectImage(param, obj, outparamkey)
- elif param == 'ram': # SetParameterValue in OTB<7.4 doesn't work for ram parameter cf gitlab OTB issue 2200
- self.app.SetParameterInt('ram', int(obj))
- elif not isinstance(obj, list): # any other parameters (str, int...)
- self.app.SetParameterValue(param, obj)
- # Images list
- elif self.__is_key_images_list(param):
- # To enable possible in-memory connections, we go through the list and set the parameters one by one
- for inp in obj:
- if isinstance(inp, otbObject):
- self.app.ConnectImage(param, inp.app, inp.output_param)
- elif isinstance(inp, otb.Application): # this is for backward comp with plain OTB
- outparamkey = [param for param in inp.GetParametersKeys() if
- inp.GetParameterType(param) == otb.ParameterType_OutputImage][0]
- self.app.ConnectImage(param, inp, outparamkey)
- else: # here `input` should be an image filepath
- # Append `input` to the list, do not overwrite any previously set element of the image list
- self.app.AddParameterStringList(param, inp)
- # List of any other types (str, int...)
- else:
- self.app.SetParameterValue(param, obj)
+ if obj is not None:
+ # Single-parameter cases
+ if isinstance(obj, otbObject):
+ self.app.ConnectImage(param, obj.app, obj.output_param)
+ elif isinstance(obj, otb.Application): # this is for backward comp with plain OTB
+ outparamkey = [param for param in obj.GetParametersKeys()
+ if obj.GetParameterType(param) == otb.ParameterType_OutputImage][0]
+ self.app.ConnectImage(param, obj, outparamkey)
+ elif param == 'ram': # SetParameterValue in OTB<7.4 doesn't work for ram parameter cf gitlab OTB issue 2200
+ self.app.SetParameterInt('ram', int(obj))
+ elif not isinstance(obj, list): # any other parameters (str, int...)
+ self.app.SetParameterValue(param, obj)
+ # Images list
+ elif self.__is_key_images_list(param):
+ # To enable possible in-memory connections, we go through the list and set the parameters one by one
+ for inp in obj:
+ if isinstance(inp, otbObject):
+ self.app.ConnectImage(param, inp.app, inp.output_param)
+ elif isinstance(inp, otb.Application): # this is for backward comp with plain OTB
+ outparamkey = [param for param in inp.GetParametersKeys() if
+ inp.GetParameterType(param) == otb.ParameterType_OutputImage][0]
+ self.app.ConnectImage(param, inp, outparamkey)
+ else: # here `input` should be an image filepath
+ # Append `input` to the list, do not overwrite any previously set element of the image list
+ self.app.AddParameterStringList(param, inp)
+ # List of any other types (str, int...)
+ else:
+ self.app.SetParameterValue(param, obj)
def __propagate_pixel_type(self):
"""
--
GitLab
From 87eb8dd7a6365d57f265ab9dabfdb24b3f5d90d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20Nar=C3=A7on?= <nicolas.narcon@inrae.fr>
Date: Mon, 4 Jul 2022 18:15:17 +0200
Subject: [PATCH 19/22] ENH: add an allow to failure for some tests
---
tests/pipeline_test.py | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
diff --git a/tests/pipeline_test.py b/tests/pipeline_test.py
index 88c8212..1e02268 100644
--- a/tests/pipeline_test.py
+++ b/tests/pipeline_test.py
@@ -21,6 +21,10 @@ PYOTB_BLOCKS = [
ALL_BLOCKS = PYOTB_BLOCKS + OTBAPPS_BLOCKS
+# These apps are problematic when used in pipelines with intermediate outputs
+# (cf https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/2290)
+PROBLEMATIC_APPS = ['DynamicConvert', 'BandMath']
+
def backward():
"""
@@ -175,10 +179,16 @@ for pipeline, errs in results.items():
msg = f"\033[91m{msg}\033[0m"
msg += " | "
if has_err:
- nb_fails += 1
causes = [f"{section}: " + ", ".join([f"app{i}" for i in out_ids])
for section, out_ids in errs.items() if out_ids]
msg += "\033[91mFAIL\033[0m (" + "; ".join(causes) + ")"
+
+ # There is a failure when the pipeline length is >=3, the last app is an Operation and the first app of the
+ # piepline is one of the problematic apps
+ if ("write" in args and "backward" not in args and isinstance(pipeline[-1], pyotb.Operation)
+ and len(pipeline) == 3 and pipeline[0].name in PROBLEMATIC_APPS and pipeline[1].name not in PROBLEMATIC_APPS):
+ continue
+ nb_fails += 1
else:
msg += "\033[92mPASS\033[0m"
print(msg)
--
GitLab
From 807ddca251ca07c952a360e51687a9429eaaa727 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20Nar=C3=A7on?= <nicolas.narcon@inrae.fr>
Date: Mon, 4 Jul 2022 18:36:08 +0200
Subject: [PATCH 20/22] FIX: pipelines tests
---
tests/pipeline_test.py | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/tests/pipeline_test.py b/tests/pipeline_test.py
index 1e02268..a3d7ff4 100644
--- a/tests/pipeline_test.py
+++ b/tests/pipeline_test.py
@@ -171,6 +171,7 @@ print(f'Tests summary (\033[93mTest options: {"; ".join(args)}\033[0m)')
print("Pipeline".ljust(cols) + " | Status (reason)")
print("-" * cols + "-|-" + "-" * 20)
nb_fails = 0
+allowed_to_fail = 0
for pipeline, errs in results.items():
has_err = sum(len(value) for key, value in errs.items()) > 0
graph = pipeline2str(pipeline)
@@ -186,11 +187,12 @@ for pipeline, errs in results.items():
# There is a failure when the pipeline length is >=3, the last app is an Operation and the first app of the
# piepline is one of the problematic apps
if ("write" in args and "backward" not in args and isinstance(pipeline[-1], pyotb.Operation)
- and len(pipeline) == 3 and pipeline[0].name in PROBLEMATIC_APPS and pipeline[1].name not in PROBLEMATIC_APPS):
- continue
- nb_fails += 1
+ and len(pipeline) == 3 and pipeline[0].name in PROBLEMATIC_APPS):
+ allowed_to_fail += 1
+ else:
+ nb_fails += 1
else:
msg += "\033[92mPASS\033[0m"
print(msg)
-print(f"End of summary ({nb_fails} error(s)).", flush=True)
+print(f"End of summary ({nb_fails} error(s), {allowed_to_fail} 'allowed to fail' error(s))", flush=True)
assert nb_fails == 0, "One of the pipelines have failed. Please read the report."
--
GitLab
From 2d9195a3f9d25e8d74b50e535caeb3ec218d8a06 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20Nar=C3=A7on?= <nicolas.narcon@inrae.fr>
Date: Tue, 5 Jul 2022 07:38:59 +0200
Subject: [PATCH 21/22] ENH: remove commented piece of code
---
pyotb/core.py | 4 ----
1 file changed, 4 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 5ef8c6c..791f088 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -606,10 +606,6 @@ class App(otbObject):
dtypes = {key: parse_pixel_type(pixel_type) for key in self.output_parameters_keys}
for key, typ in dtypes.items():
self.app.SetParameterOutputImagePixelType(key, typ)
- # Here we make sure that intermediate outputs will be flushed to disk
- if self.__with_output():
- pass
- # self.app.PropagateConnectMode(True)
# Run app, write output if needed, update `finished` property
if execute or not self.output_param:
self.execute()
--
GitLab
From aa8ea2c5b96c95d8408a71d6ec88df699653ba3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20Nar=C3=A7on?= <nicolas.narcon@inrae.fr>
Date: Tue, 5 Jul 2022 07:47:17 +0200
Subject: [PATCH 22/22] STYLE: bump version
---
README.md | 2 +-
doc/installation.md | 2 +-
doc/troubleshooting.md | 4 +++-
pyotb/__init__.py | 2 +-
setup.py | 2 +-
5 files changed, 7 insertions(+), 5 deletions(-)
diff --git a/README.md b/README.md
index 9879d45..6f0adcd 100644
--- a/README.md
+++ b/README.md
@@ -11,7 +11,7 @@ Requirements:
pip install pyotb --upgrade
```
-For Python>=3.6, latest version available is pyotb 1.4.0 For Python 3.5, latest version available is pyotb 1.2.2
+For Python>=3.6, latest version available is pyotb 1.4.1 For Python 3.5, latest version available is pyotb 1.2.2
## Quickstart: running an OTB application as a oneliner
pyotb has been written so that it is more convenient to run an application in Python.
diff --git a/doc/installation.md b/doc/installation.md
index 991ecb9..6c79ef8 100644
--- a/doc/installation.md
+++ b/doc/installation.md
@@ -8,4 +8,4 @@
pip install pyotb --upgrade
```
-For Python>=3.6, latest version available is pyotb 1.4.0. For Python 3.5, latest version available is pyotb 1.2.2
+For Python>=3.6, latest version available is pyotb 1.4.1. For Python 3.5, latest version available is pyotb 1.2.2
diff --git a/doc/troubleshooting.md b/doc/troubleshooting.md
index aae8be6..e6b0b4c 100644
--- a/doc/troubleshooting.md
+++ b/doc/troubleshooting.md
@@ -54,7 +54,7 @@ one_band.write('one_band.tif')
#### Example of failures involving arithmetic operation
-One can meet errors when using arithmetic operations raster at the end of a pipeline when DynamicConvert, BandMath or
+One can meet errors when using arithmetic operations at the end of a pipeline when DynamicConvert, BandMath or
OpticalCalibration is involved:
```python
@@ -69,3 +69,5 @@ inp.write('one_band.tif')
inp_new.write('one_band_nodata.tif') # Failure here
absolute.write('absolute.tif') # Failure here
```
+
+When writing only the final result, i.e. the end of the pipeline (`absolute.write('absolute.tif')`), there is no problem.
diff --git a/pyotb/__init__.py b/pyotb/__init__.py
index a57a623..561ef64 100644
--- a/pyotb/__init__.py
+++ b/pyotb/__init__.py
@@ -2,7 +2,7 @@
"""
This module provides convenient python wrapping of otbApplications
"""
-__version__ = "1.4.0"
+__version__ = "1.4.1"
from .apps import *
from .core import App, Output, Input, get_nbchannels, get_pixel_type
diff --git a/setup.py b/setup.py
index 7785ddc..c99a422 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
setuptools.setup(
name="pyotb",
- version="1.4.0",
+ version="1.4.1",
author="Nicolas Narçon",
author_email="nicolas.narcon@gmail.com",
description="Library to enable easy use of the Orfeo Tool Box (OTB) in Python",
--
GitLab