From 8f0fe4b8dc1b406ea26108ed3a77b8ee521e533b Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@irstea.fr> Date: Tue, 28 Jun 2022 15:33:14 +0200 Subject: [PATCH 01/22] TEST: forward/backward, with/without intermediate outputs --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a6610b0..1eecadb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -106,3 +106,5 @@ pipeline_test: - cd tests - python3 pipeline_test.py - python3 pipeline_test.py backward + - python3 pipeline_test.py no-intermediate-result + - python3 pipeline_test.py backward no-intermediate-result -- GitLab From 229a43c014a54ae09d269f889b43ff46aedc7f1c Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@irstea.fr> Date: Tue, 28 Jun 2022 15:33:18 +0200 Subject: [PATCH 02/22] TEST: forward/backward, with/without intermediate outputs --- tests/pipeline_test.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/pipeline_test.py b/tests/pipeline_test.py index 3b152b5..327f110 100644 --- a/tests/pipeline_test.py +++ b/tests/pipeline_test.py @@ -101,15 +101,16 @@ def test_pipeline(pipeline): pipeline: pipeline (list of pyotb objects) """ + args = [arg.lower() for arg in sys.argv[1:]] if len(sys.argv) > 1 else [] report = {"shapes_errs": [], "write_errs": []} # Test outputs shapes - generator = enumerate(pipeline) - if len(sys.argv) > 1: - if "backward" in sys.argv[1].lower(): - print("Perform tests in backward mode") - generator = enumerate(reversed(pipeline)) - for i, app in generator: + pipeline_items = [pipeline[-1]] if "no-intermediate-output" in args else pipeline + generator = lambda: enumerate(pipeline_items) + if "backward" in args: + print("Perform tests in backward mode") + generator = lambda: enumerate(reversed(pipeline_items)) + for i, app in generator(): try: print(f"Trying to access shape of app {app.name} output...") shape = app.shape @@ -120,7 +121,7 @@ def test_pipeline(pipeline): report["shapes_errs"].append(i) # Test all pipeline outputs - for i, app in generator: + for i, app in generator(): if not check_app_write(app, f"/tmp/out_{i}.tif"): report["write_errs"].append(i) @@ -174,7 +175,7 @@ for pipeline, errs in results.items(): msg += " | " if has_err: nb_fails += 1 - causes = [f"{section}: " + ", ".join([str(i) for i in out_ids]) + causes = [f"{section}: " + ", ".join([f"app{i}" for i in out_ids]) for section, out_ids in errs.items() if out_ids] msg += "\033[91mFAIL\033[0m (" + "; ".join(causes) + ")" else: -- GitLab From 919f7c918eeb1d38481915ee6bc56da8557575a3 Mon Sep 17 00:00:00 2001 From: Narcon Nicolas <nicolas.narcon@inrae.fr> Date: Wed, 29 Jun 2022 14:12:51 +0200 Subject: [PATCH 03/22] WIP: remove execute in Operation init --- pyotb/core.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 598185a..a3d91e8 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -802,7 +802,7 @@ class App(otbObject): try: self.app.Execute() if self.__with_output(): - self.app.WriteOutput() + self.app.ExecuteAndWriteOutput() self.finished = True except (RuntimeError, FileNotFoundError) as e: raise Exception(f'{self.name}: error during during app execution') from e @@ -1101,7 +1101,6 @@ class Operation(otbObject): app = App('BandMath', il=self.unique_inputs, exp=self.exp) else: app = App('BandMathX', il=self.unique_inputs, exp=self.exp) - app.execute() self.app = app.app def create_fake_exp(self, operator, inputs, nb_bands=None): -- GitLab From 97ef39a2ed4f27e05bdfd6f48c5af5f3b7b045e6 Mon Sep 17 00:00:00 2001 From: Narcon Nicolas <nicolas.narcon@inrae.fr> Date: Wed, 29 Jun 2022 14:44:46 +0200 Subject: [PATCH 04/22] WIP: loosen the condition for execution of objects in pipelines --- doc/index.md | 3 ++- doc/interaction.md | 3 +++ pyotb/core.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/index.md b/doc/index.md index eb936d5..d73bd16 100644 --- a/doc/index.md +++ b/doc/index.md @@ -1,4 +1,4 @@ -# Pyotb +# Pyotb: Orfeo Toolbox for Python pyotb is a Python extension of Orfeo Toolbox. It has been built on top of the existing Python API of OTB, in order to make OTB more Python friendly. @@ -20,4 +20,5 @@ to make OTB more Python friendly. - [Comparison between pyotb and OTB native library](comparison_otb.md) - [OTB versions](otb_versions.md) - [Managing loggers](managing_loggers.md) + ## API diff --git a/doc/interaction.md b/doc/interaction.md index e8493a8..6c87417 100644 --- a/doc/interaction.md +++ b/doc/interaction.md @@ -33,6 +33,7 @@ noisy_image = inp + white_noise # magic: this is a pyotb object that has the sa noisy_image.write('image_plus_noise.tif') ``` Limitations : + - The whole image is loaded into memory - The georeference can not be modified. Thus, numpy operations can not change the image or pixel size @@ -70,9 +71,11 @@ res = scalar_product('image1.tif', 'image2.tif') # magic: this is a pyotb objec ``` Advantages : + - The process supports streaming, hence the whole image is **not** loaded into memory - Can be integrated in OTB pipelines Limitations : + - It is not possible to use the tensorflow python API inside a script where OTBTF is used because of compilation issues between Tensorflow and OTBTF, i.e. `import tensorflow` doesn't work in a script where OTBTF apps have been initialized diff --git a/pyotb/core.py b/pyotb/core.py index a3d91e8..f1418ca 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -176,7 +176,7 @@ class otbObject(ABC): if isinstance(self, App): if not self.finished: self.execute() - elif isinstance(self, Output): + elif isinstance(self, otbObject): self.app.Execute() # Special methods -- GitLab From 2ce882f1b00e034de7983fb0829089323e9fa0ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr> Date: Wed, 29 Jun 2022 15:08:18 +0000 Subject: [PATCH 05/22] Fix test pipelines bugs --- .gitlab-ci.yml | 19 ++++++++++++------- tests/pipeline_test.py | 31 ++++++++++++++++--------------- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1eecadb..64ab7dd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -83,6 +83,7 @@ pages: allow_failure: false before_script: - pip install . + - cd tests variables: OTB_ROOT: /opt/otb LD_LIBRARY_PATH: /opt/otb/lib @@ -97,14 +98,18 @@ import_pyotb: compute_ndvi: extends: .test_base script: - - cd tests - python3 ndvi_test.py -pipeline_test: +pipeline_tests: extends: .test_base script: - - cd tests - - python3 pipeline_test.py - - python3 pipeline_test.py backward - - python3 pipeline_test.py no-intermediate-result - - python3 pipeline_test.py backward no-intermediate-result + - export OTB_LOGGER_LEVEL="ERROR" + - export PYOTB_LOGGER_LEVEL="ERROR" + - python3 pipeline_test.py shape + - python3 pipeline_test.py shape backward + - python3 pipeline_test.py shape no-intermediate-result + - python3 pipeline_test.py shape backward no-intermediate-result + - python3 pipeline_test.py write + - python3 pipeline_test.py write backward + - python3 pipeline_test.py write no-intermediate-result + - python3 pipeline_test.py write backward no-intermediate-result diff --git a/tests/pipeline_test.py b/tests/pipeline_test.py index 327f110..88c8212 100644 --- a/tests/pipeline_test.py +++ b/tests/pipeline_test.py @@ -52,7 +52,7 @@ def check_app_write(app, out): filepath = 'Data/Input/QB_MUL_ROI_1000_100.tif' pyotb_input = pyotb.Input(filepath) - +args = [arg.lower() for arg in sys.argv[1:]] if len(sys.argv) > 1 else [] def generate_pipeline(inp, building_blocks): """ @@ -101,7 +101,6 @@ def test_pipeline(pipeline): pipeline: pipeline (list of pyotb objects) """ - args = [arg.lower() for arg in sys.argv[1:]] if len(sys.argv) > 1 else [] report = {"shapes_errs": [], "write_errs": []} # Test outputs shapes @@ -110,20 +109,22 @@ def test_pipeline(pipeline): if "backward" in args: print("Perform tests in backward mode") generator = lambda: enumerate(reversed(pipeline_items)) - for i, app in generator(): - try: - print(f"Trying to access shape of app {app.name} output...") - shape = app.shape - print(f"App {app.name} output shape is {shape}") - except Exception as e: - print("\n\033[91mGET SHAPE ERROR\033[0m") - print(e) - report["shapes_errs"].append(i) + if "shape" in args: + for i, app in generator(): + try: + print(f"Trying to access shape of app {app.name} output...") + shape = app.shape + print(f"App {app.name} output shape is {shape}") + except Exception as e: + print("\n\033[91mGET SHAPE ERROR\033[0m") + print(e) + report["shapes_errs"].append(i) # Test all pipeline outputs - for i, app in generator(): - if not check_app_write(app, f"/tmp/out_{i}.tif"): - report["write_errs"].append(i) + if "write" in args: + for i, app in generator(): + if not check_app_write(app, f"/tmp/out_{i}.tif"): + report["write_errs"].append(i) return report @@ -162,7 +163,7 @@ for pipeline in pipelines: # Summary cols = max([len(pipeline2str(pipeline)) for pipeline in pipelines]) + 1 -print("Tests summary:") +print(f'Tests summary (\033[93mTest options: {"; ".join(args)}\033[0m)') print("Pipeline".ljust(cols) + " | Status (reason)") print("-" * cols + "-|-" + "-" * 20) nb_fails = 0 -- GitLab From e93d1f4a0fc28a338085832a1117e314fcf9afdd Mon Sep 17 00:00:00 2001 From: Narcon Nicolas <nicolas.narcon@inrae.fr> Date: Wed, 29 Jun 2022 17:10:09 +0200 Subject: [PATCH 06/22] WIP: removing all PropagateConnectMode(False) --- pyotb/core.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index f1418ca..c83677c 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -120,8 +120,6 @@ class otbObject(ABC): if key in dtypes: self.app.SetParameterOutputImagePixelType(key, dtypes[key]) - self.app.PropagateConnectMode(False) - if isinstance(self, App): return self.execute() @@ -781,7 +779,8 @@ class App(otbObject): self.app.SetParameterOutputImagePixelType(key, typ) # Here we make sure that intermediate outputs will be flushed to disk if self.__with_output(): - self.app.PropagateConnectMode(False) + pass + # self.app.PropagateConnectMode(True) # Run app, write output if needed, update `finished` property if execute or not self.output_param: self.execute() -- GitLab From ede24c33b0a752d382b069148fdc4e2f7f93227e Mon Sep 17 00:00:00 2001 From: Narcon Nicolas <nicolas.narcon@inrae.fr> Date: Wed, 29 Jun 2022 17:36:29 +0200 Subject: [PATCH 07/22] WIP: keep reference to pyotb app as attribute for Slicer and Operation --- pyotb/core.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index c83677c..2003cae 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -175,7 +175,8 @@ class otbObject(ABC): if not self.finished: self.execute() elif isinstance(self, otbObject): - self.app.Execute() + if not self.pyotb_app.finished: + self.pyotb_app.execute() # Special methods def __getitem__(self, key): @@ -615,8 +616,8 @@ class Slicer(otbObject): # Execute app app.set_parameters(**parameters) app.execute() - # Keeping the OTB app, not the pyotb app - self.app = app.app + # Keeping the OTB app and the pyotb app + self.pyotb_app, self.app = app, app.app # These are some attributes when the user simply wants to extract *one* band to be used in an Operation if not spatial_slicing and isinstance(channels, list) and len(channels) == 1: @@ -1100,7 +1101,9 @@ class Operation(otbObject): app = App('BandMath', il=self.unique_inputs, exp=self.exp) else: app = App('BandMathX', il=self.unique_inputs, exp=self.exp) - self.app = app.app + + # Keeping the OTB app and the pyotb app + self.pyotb_app, self.app = app, app.app def create_fake_exp(self, operator, inputs, nb_bands=None): """ -- GitLab From a47be77df3d06d6789053d42d583f5a983ab9dca Mon Sep 17 00:00:00 2001 From: Narcon Nicolas <nicolas.narcon@inrae.fr> Date: Wed, 29 Jun 2022 17:45:32 +0200 Subject: [PATCH 08/22] WIP: keep reference to pyotb app as attribute for Input and Output --- pyotb/core.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 2003cae..7d198b4 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -640,7 +640,9 @@ class Input(otbObject): self.filepath = filepath self.name = f'Input from {filepath}' app = App('ExtractROI', filepath, execute=True, propagate_pixel_type=True) - self.app = app.app + + # Keeping the OTB app and the pyotb app + self.pyotb_app, self.app = app, app.app def __str__(self): """ @@ -656,14 +658,15 @@ class Output(otbObject): Class for output of an app """ - def __init__(self, app, output_parameter_key): + def __init__(self, pyotb_app, output_parameter_key): """ Args: - app: The OTB application + app: The pyotb App output_parameter_key: Output parameter key """ - self.app = app # keeping a reference of the OTB app + # Keeping the OTB app and the pyotb app + self.pyotb_app, self.app = app, app.app self.output_parameter_key = output_parameter_key self.name = f'Output {output_parameter_key} from {self.app.GetName()}' @@ -992,7 +995,7 @@ class App(otbObject): """ for key in self.app.GetParametersKeys(): if key in self.output_parameters_keys: # raster outputs - output = Output(self.app, key) + output = Output(self, key) setattr(self, key, output) elif key not in ('parameters',): # any other attributes (scalars...) try: -- GitLab From 1e5608336b4ff291a12cb8124c05b9853280393e Mon Sep 17 00:00:00 2001 From: Narcon Nicolas <nicolas.narcon@inrae.fr> Date: Wed, 29 Jun 2022 17:52:42 +0200 Subject: [PATCH 09/22] FIX: typo --- pyotb/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index 7d198b4..9e6926f 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -666,7 +666,7 @@ class Output(otbObject): """ # Keeping the OTB app and the pyotb app - self.pyotb_app, self.app = app, app.app + self.pyotb_app, self.app = pyotb_app, pyotb_app.app self.output_parameter_key = output_parameter_key self.name = f'Output {output_parameter_key} from {self.app.GetName()}' -- GitLab From 3cb32c65ff933a2341ef390dd2f72a50c7cc4b56 Mon Sep 17 00:00:00 2001 From: Narcon Nicolas <nicolas.narcon@inrae.fr> Date: Wed, 29 Jun 2022 18:09:51 +0200 Subject: [PATCH 10/22] FIX: remove execute in Slicer --- pyotb/core.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 9e6926f..f35a465 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -570,8 +570,6 @@ class Slicer(otbObject): # Trigger source app execution if needed x.execute_if_needed() app = App('ExtractROI', {'in': x, 'mode': 'extent'}, propagate_pixel_type=True) - # First exec required in order to read image dim - app.app.Execute() parameters = {} # Channel slicing @@ -615,7 +613,7 @@ class Slicer(otbObject): spatial_slicing = True # Execute app app.set_parameters(**parameters) - app.execute() + # Keeping the OTB app and the pyotb app self.pyotb_app, self.app = app, app.app -- GitLab From 34f4a4499fc61a208c91ac131f22598dcea2069e Mon Sep 17 00:00:00 2001 From: Narcon Nicolas <nicolas.narcon@inrae.fr> Date: Thu, 30 Jun 2022 09:08:29 +0200 Subject: [PATCH 11/22] FIX: add execute in Slicer --- pyotb/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyotb/core.py b/pyotb/core.py index f35a465..97e91e8 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -570,6 +570,7 @@ class Slicer(otbObject): # Trigger source app execution if needed x.execute_if_needed() app = App('ExtractROI', {'in': x, 'mode': 'extent'}, propagate_pixel_type=True) + app.app.Execute() parameters = {} # Channel slicing -- GitLab From 47b1647f1deea28d5868d7417db8deeba67fa0c2 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Thu, 30 Jun 2022 11:20:22 +0200 Subject: [PATCH 12/22] ENH: separate tests in different jobs --- .gitlab-ci.yml | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 64ab7dd..b7d75da 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -100,16 +100,43 @@ compute_ndvi: script: - python3 ndvi_test.py -pipeline_tests: +pipeline_test_shape: extends: .test_base script: - - export OTB_LOGGER_LEVEL="ERROR" - - export PYOTB_LOGGER_LEVEL="ERROR" - python3 pipeline_test.py shape + +pipeline_test_shape_backward: + extends: .test_base + script: - python3 pipeline_test.py shape backward + +pipeline_test_shape_nointermediate: + extends: .test_base + script: - python3 pipeline_test.py shape no-intermediate-result + +pipeline_test_shape_backward_nointermediate: + extends: .test_base + script: - python3 pipeline_test.py shape backward no-intermediate-result + +pipeline_test_write: + extends: .test_base + script: - python3 pipeline_test.py write + +pipeline_test_write_backward: + extends: .test_base + script: - python3 pipeline_test.py write backward + +pipeline_test_write_nointermediate: + extends: .test_base + script: - python3 pipeline_test.py write no-intermediate-result + +pipeline_test_write_backward_nointermediate: + extends: .test_base + script: - python3 pipeline_test.py write backward no-intermediate-result + -- GitLab From b9abb554b599cdc02c37e87a9d73b315c2e52442 Mon Sep 17 00:00:00 2001 From: Narcon Nicolas <nicolas.narcon@inrae.fr> Date: Thu, 30 Jun 2022 16:47:02 +0200 Subject: [PATCH 13/22] FIX: in Slicer, execute app only when slicing channels --- pyotb/core.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 97e91e8..749e364 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -567,15 +567,14 @@ class Slicer(otbObject): # Initialize the app that will be used for writing the slicer self.output_parameter_key = 'out' self.name = 'Slicer' - # Trigger source app execution if needed - x.execute_if_needed() app = App('ExtractROI', {'in': x, 'mode': 'extent'}, propagate_pixel_type=True) - app.app.Execute() parameters = {} # Channel slicing - nb_channels = get_nbchannels(x) if channels != slice(None, None, None): + # Trigger source app execution if needed + nb_channels = get_nbchannels(x) + app.app.Execute() # this is needed by ExtractROI for setting the `cl` parameter # if needed, converting int to list if isinstance(channels, int): channels = [channels] -- GitLab From de5ee0143cca2987c04ce667ca6a6021c2f30b76 Mon Sep 17 00:00:00 2001 From: Narcon Nicolas <nicolas.narcon@inrae.fr> Date: Thu, 30 Jun 2022 16:49:22 +0200 Subject: [PATCH 14/22] DOC: add troubleshooting section + change site name --- doc/index.md | 2 ++ doc/troubleshooting.md | 74 ++++++++++++++++++++++++++++++++++++++++++ mkdocs.yml | 5 +-- 3 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 doc/troubleshooting.md diff --git a/doc/index.md b/doc/index.md index d73bd16..efebb29 100644 --- a/doc/index.md +++ b/doc/index.md @@ -20,5 +20,7 @@ to make OTB more Python friendly. - [Comparison between pyotb and OTB native library](comparison_otb.md) - [OTB versions](otb_versions.md) - [Managing loggers](managing_loggers.md) +- [Troubleshooting & limitations](troubleshooting.md) + ## API diff --git a/doc/troubleshooting.md b/doc/troubleshooting.md new file mode 100644 index 0000000..f76da8e --- /dev/null +++ b/doc/troubleshooting.md @@ -0,0 +1,74 @@ +## Troubleshooting: Known limitations + +### Failure of intermediate writing + +When chaining applications in-memory, there may be some problems when writing intermediate results, depending on the order +the writings are requested. Some examples can be found below: + +#### Example of failures involving slicing + +For some applications (non-exhaustive know list: OpticalCalibration, DynamicConvert, BandMath), we can face unexpected +failures when using channels slicing +```python +import pyotb + +inp = pyotb.DynamicConvert('raster.tif') +one_band = inp[:, :, 1] + +# this works +one_band.write('one_band.tif') + +# this works +one_band.write('one_band.tif') +inp.write('stretched.tif') + +# this does not work +inp.write('stretched.tif') +one_band.write('one_band.tif') # Failure here +``` + +However, when using only spatial slicing, no issue has been reported: + +```python +import pyotb + +inp = pyotb.DynamicConvert('raster.tif') +one_band = inp[:100, :100, :] + +# this works +inp.write('stretched.tif') +one_band.write('one_band.tif') +``` + + +#### Example of failures involving arithmetic operation + +One can meet errors when using arithmetic operations on monoband raster. This limitation is related to BandMath. + +```python +import pyotb + +inp = pyotb.Input('1band_raster.tif') +absolute = abs(inp) +one_band = absolute[:, :, 0] + +# this does not work +absolute.write('absolute.tif') +one_band.write('one_band.tif') # Failure here + +``` + +However, the same code with a multibands raster doesn't raise any error + +```python +import pyotb + +inp = pyotb.Input('4bands_raster.tif') +absolute = abs(inp) +one_band = absolute[:, :, 0] + +# this works +one_band.write('one_band.tif') +absolute.write('absolute.tif') + +``` \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 7dcadf5..07c04c2 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -27,7 +27,7 @@ plugins: nav: - Home: index.md - Get Started: - - Installation: installation.md + - Installation of pyotb: installation.md - How to use pyotb: quickstart.md - Useful features: features.md - Functions: functions.md @@ -39,6 +39,7 @@ nav: - Comparison between pyotb and OTB native library: comparison_otb.md - OTB versions: otb_versions.md - Managing loggers: managing_loggers.md + - Troubleshooting & limitations: troubleshooting.md - API: - pyotb: - core: reference/pyotb/core.md @@ -69,7 +70,7 @@ markdown_extensions: - pymdownx.snippets # rest of the navigation.. -site_name: pyotb +site_name: "pyotb documentation: a Python extension of OTB" repo_url: https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb repo_name: pyotb docs_dir: doc/ -- GitLab From e918b941f8c8d41afe3c698adeacf100b80cda39 Mon Sep 17 00:00:00 2001 From: Narcon Nicolas <nicolas.narcon@inrae.fr> Date: Fri, 1 Jul 2022 11:17:02 +0200 Subject: [PATCH 15/22] DOC: more troubleshooting --- doc/troubleshooting.md | 41 +++++++++++++++++++---------------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/doc/troubleshooting.md b/doc/troubleshooting.md index f76da8e..aae8be6 100644 --- a/doc/troubleshooting.md +++ b/doc/troubleshooting.md @@ -27,7 +27,18 @@ inp.write('stretched.tif') one_band.write('one_band.tif') # Failure here ``` -However, when using only spatial slicing, no issue has been reported: +When writing is triggered right after the application declaration, no problem occurs: +```python +import pyotb + +inp = pyotb.DynamicConvert('raster.tif') +inp.write('stretched.tif') + +one_band = inp[:, :, 1] +one_band.write('one_band.tif') # no failure +``` + +Also, when using only spatial slicing, no issue has been reported: ```python import pyotb @@ -43,32 +54,18 @@ one_band.write('one_band.tif') #### Example of failures involving arithmetic operation -One can meet errors when using arithmetic operations on monoband raster. This limitation is related to BandMath. +One can meet errors when using arithmetic operations raster at the end of a pipeline when DynamicConvert, BandMath or +OpticalCalibration is involved: ```python import pyotb -inp = pyotb.Input('1band_raster.tif') +inp = pyotb.DynamicConvert('raster.tif') +inp_new = pyotb.ManageNoData(inp, mode='changevalue') absolute = abs(inp) -one_band = absolute[:, :, 0] # this does not work -absolute.write('absolute.tif') -one_band.write('one_band.tif') # Failure here - +inp.write('one_band.tif') +inp_new.write('one_band_nodata.tif') # Failure here +absolute.write('absolute.tif') # Failure here ``` - -However, the same code with a multibands raster doesn't raise any error - -```python -import pyotb - -inp = pyotb.Input('4bands_raster.tif') -absolute = abs(inp) -one_band = absolute[:, :, 0] - -# this works -one_band.write('one_band.tif') -absolute.write('absolute.tif') - -``` \ No newline at end of file -- GitLab From 35c4d6dbeda3178df09d5a0d5be87c145f19563a Mon Sep 17 00:00:00 2001 From: Narcon Nicolas <nicolas.narcon@inrae.fr> Date: Fri, 1 Jul 2022 11:18:17 +0200 Subject: [PATCH 16/22] REFAC: reorder classes --- pyotb/core.py | 460 +++++++++++++++++++++++++------------------------- 1 file changed, 226 insertions(+), 234 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 749e364..b13c7d4 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -548,185 +548,12 @@ class otbObject(ABC): return NotImplemented -class Slicer(otbObject): - """Slicer objects i.e. when we call something like raster[:, :, 2] from Python""" - - def __init__(self, x, rows, cols, channels): - """ - Create a slicer object, that can be used directly for writing or inside a BandMath : - - an ExtractROI app that handles extracting bands and ROI and can be written to disk or used in pipelines - - in case the user only wants to extract one band, an expression such as "im1b#" - - Args: - x: input - rows: rows slicing (e.g. 100:2000) - cols: columns slicing (e.g. 100:2000) - channels: channels, can be slicing, list or int - - """ - # Initialize the app that will be used for writing the slicer - self.output_parameter_key = 'out' - self.name = 'Slicer' - app = App('ExtractROI', {'in': x, 'mode': 'extent'}, propagate_pixel_type=True) - - parameters = {} - # Channel slicing - if channels != slice(None, None, None): - # Trigger source app execution if needed - nb_channels = get_nbchannels(x) - app.app.Execute() # this is needed by ExtractROI for setting the `cl` parameter - # if needed, converting int to list - if isinstance(channels, int): - channels = [channels] - # if needed, converting slice to list - elif isinstance(channels, slice): - channels_start = channels.start if channels.start is not None else 0 - channels_end = channels.stop if channels.stop is not None else nb_channels - channels_step = channels.step if channels.step is not None else 1 - channels = range(channels_start, channels_end, channels_step) - elif isinstance(channels, tuple): - channels = list(channels) - elif not isinstance(channels, list): - raise ValueError(f'Invalid type for channels, should be int, slice or list of bands. : {channels}') - - # Change the potential negative index values to reverse index - channels = [c if c >= 0 else nb_channels + c for c in channels] - parameters.update({'cl': [f'Channel{i + 1}' for i in channels]}) - - # Spatial slicing - spatial_slicing = False - # TODO: handle PixelValue app so that accessing value is possible, e.g. raster[120, 200, 0] - # TODO TBD: handle the step value in the slice so that NN undersampling is possible ? e.g. obj[::2, ::2] - if rows.start is not None: - parameters.update({'mode.extent.uly': rows.start}) - spatial_slicing = True - if rows.stop is not None and rows.stop != -1: - parameters.update( - {'mode.extent.lry': rows.stop - 1}) # subtract 1 to be compliant with python convention - spatial_slicing = True - if cols.start is not None: - parameters.update({'mode.extent.ulx': cols.start}) - spatial_slicing = True - if cols.stop is not None and cols.stop != -1: - parameters.update( - {'mode.extent.lrx': cols.stop - 1}) # subtract 1 to be compliant with python convention - spatial_slicing = True - # Execute app - app.set_parameters(**parameters) - - # Keeping the OTB app and the pyotb app - self.pyotb_app, self.app = app, app.app - - # These are some attributes when the user simply wants to extract *one* band to be used in an Operation - if not spatial_slicing and isinstance(channels, list) and len(channels) == 1: - self.one_band_sliced = channels[0] + 1 # OTB convention: channels start at 1 - self.input = x - - -class Input(otbObject): - """ - Class for transforming a filepath to pyOTB object - """ - - def __init__(self, filepath): - """ - Args: - filepath: raster file path - - """ - self.output_parameter_key = 'out' - self.filepath = filepath - self.name = f'Input from {filepath}' - app = App('ExtractROI', filepath, execute=True, propagate_pixel_type=True) - - # Keeping the OTB app and the pyotb app - self.pyotb_app, self.app = app, app.app - - def __str__(self): - """ - Returns: - string representation - - """ - return f'<pyotb.Input object from {self.filepath}>' - - -class Output(otbObject): - """ - Class for output of an app - """ - - def __init__(self, pyotb_app, output_parameter_key): - """ - Args: - app: The pyotb App - output_parameter_key: Output parameter key - - """ - # Keeping the OTB app and the pyotb app - self.pyotb_app, self.app = pyotb_app, pyotb_app.app - self.output_parameter_key = output_parameter_key - self.name = f'Output {output_parameter_key} from {self.app.GetName()}' - - def __str__(self): - """ - Returns: - string representation - - """ - return f'<pyotb.Output {self.app.GetName()} object, id {id(self)}>' - - class App(otbObject): """ Class of an OTB app """ _name = "" - @property - def name(self): - """ - Returns: - user's defined name or appname - - """ - return self._name or self.appname - - @name.setter - def name(self, val): - """Set custom App name - - Args: - val: new name - - """ - self._name = val - - @property - def finished(self): - """ - Property to store whether App has been executed but False if any output file is missing - - Returns: - True if exec ended and output files are found else False - - """ - if self._ended and self.find_output(): - return True - return False - - @finished.setter - def finished(self, val): - """ - Value `_ended` will be set to True right after App.execute() or App.write(), - then find_output() is called when accessing the property - - Args: - val: True if execution ended without exceptions - - """ - self._ended = val - def __init__(self, appname, *args, execute=False, image_dic=None, otb_stdout=True, pixel_type=None, propagate_pixel_type=False, **kwargs): """ @@ -790,6 +617,54 @@ class App(otbObject): else: self.__save_objects() + def get_output_parameters_keys(self): + """Get raster output parameter keys + + Returns: + output parameters keys + """ + return [param for param in self.app.GetParametersKeys() + if self.app.GetParameterType(param) == otb.ParameterType_OutputImage] + + def set_parameters(self, *args, **kwargs): + """Set some parameters of the app. When useful, e.g. for images list, this function appends the parameters + instead of overwriting them. Handles any parameters, i.e. in-memory & filepaths + + Args: + *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved + (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit") + - string, App or Output, useful when the user implicitly wants to set the param "in" + - list, useful when the user implicitly wants to set the param "il" + **kwargs: keyword arguments e.g. il=['input1.tif', oApp_object2, App_object3.out], out='output.tif' + + Raises: + Exception: when the setting of a parameter failed + + """ + parameters = kwargs + parameters.update(self.__parse_args(args)) + # Going through all arguments + for param, obj in parameters.items(): + if param not in self.app.GetParametersKeys(): + raise Exception(f"{self.name}: parameter '{param}' was not recognized. " + f"Available keys are {self.app.GetParametersKeys()}") + # When the parameter expects a list, if needed, change the value to list + if self.__is_key_list(param) and not isinstance(obj, (list, tuple)): + parameters[param] = [obj] + obj = [obj] + logger.warning('%s: Argument for parameter "%s" was converted to list', self.name, param) + try: + # This is when we actually call self.app.SetParameter* + self.__set_param(param, obj) + except (RuntimeError, TypeError, ValueError, KeyError) as e: + raise Exception(f"{self.name}: something went wrong before execution " + f"(while setting parameter {param} to '{obj}')") from e + + # Update App's parameters attribute + self.parameters.update(parameters) + if self.preserve_dtype: + self.__propagate_pixel_type() + def execute(self): """ Execute with appropriate and outputs to disk if any output parameter was set @@ -798,7 +673,6 @@ class App(otbObject): boolean flag that indicate if command executed with success """ - success = False logger.debug("%s: run execute() with parameters=%s", self.name, self.parameters) try: self.app.Execute() @@ -814,6 +688,50 @@ class App(otbObject): return success + @property + def name(self): + """ + Returns: + user's defined name or appname + + """ + return self._name or self.appname + + @name.setter + def name(self, val): + """Set custom App name + + Args: + val: new name + + """ + self._name = val + + @property + def finished(self): + """ + Property to store whether App has been executed but False if any output file is missing + + Returns: + True if exec ended and output files are found else False + + """ + if self._ended and self.find_output(): + return True + return False + + @finished.setter + def finished(self, val): + """ + Value `_ended` will be set to True right after App.execute() or App.write(), + then find_output() is called when accessing the property + + Args: + val: True if execution ended without exceptions + + """ + self._ended = val + def find_output(self): """ Find output files on disk using parameters @@ -865,54 +783,6 @@ class App(otbObject): if memory: self.app.FreeRessources() - def get_output_parameters_keys(self): - """Get raster output parameter keys - - Returns: - output parameters keys - """ - return [param for param in self.app.GetParametersKeys() - if self.app.GetParameterType(param) == otb.ParameterType_OutputImage] - - def set_parameters(self, *args, **kwargs): - """Set some parameters of the app. When useful, e.g. for images list, this function appends the parameters - instead of overwriting them. Handles any parameters, i.e. in-memory & filepaths - - Args: - *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved - (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit") - - string, App or Output, useful when the user implicitly wants to set the param "in" - - list, useful when the user implicitly wants to set the param "il" - **kwargs: keyword arguments e.g. il=['input1.tif', oApp_object2, App_object3.out], out='output.tif' - - Raises: - Exception: when the setting of a parameter failed - - """ - parameters = kwargs - parameters.update(self.__parse_args(args)) - # Going through all arguments - for param, obj in parameters.items(): - if param not in self.app.GetParametersKeys(): - raise Exception(f"{self.name}: parameter '{param}' was not recognized. " - f"Available keys are {self.app.GetParametersKeys()}") - # When the parameter expects a list, if needed, change the value to list - if self.__is_key_list(param) and not isinstance(obj, (list, tuple)): - parameters[param] = [obj] - obj = [obj] - logger.warning('%s: Argument for parameter "%s" was converted to list', self.name, param) - try: - # This is when we actually call self.app.SetParameter* - self.__set_param(param, obj) - except (RuntimeError, TypeError, ValueError, KeyError) as e: - raise Exception(f"{self.name}: something went wrong before execution " - f"(while setting parameter {param} to '{obj}')") from e - - # Update App's parameters attribute - self.parameters.update(parameters) - if self.preserve_dtype: - self.__propagate_pixel_type() - # Private functions @staticmethod def __parse_args(args): @@ -1005,22 +875,15 @@ class App(otbObject): """ Check if a key of the App is an input parameter list """ - return self.app.GetParameterType(key) in ( - otb.ParameterType_InputImageList, - otb.ParameterType_StringList, - otb.ParameterType_InputFilenameList, - otb.ParameterType_InputVectorDataList, - otb.ParameterType_ListView - ) + return self.app.GetParameterType(key) in (otb.ParameterType_InputImageList, otb.ParameterType_StringList, + otb.ParameterType_InputFilenameList, otb.ParameterType_ListView, + otb.ParameterType_InputVectorDataList) def __is_key_images_list(self, key): """ Check if a key of the App is an input parameter image list """ - return self.app.GetParameterType(key) in ( - otb.ParameterType_InputImageList, - otb.ParameterType_InputFilenameList - ) + return self.app.GetParameterType(key) in (otb.ParameterType_InputImageList, otb.ParameterType_InputFilenameList) # Special methods def __str__(self): @@ -1030,6 +893,135 @@ class App(otbObject): return f'<pyotb.App {self.appname} object id {id(self)}>' +class Slicer(otbObject): + """Slicer objects i.e. when we call something like raster[:, :, 2] from Python""" + + def __init__(self, x, rows, cols, channels): + """ + Create a slicer object, that can be used directly for writing or inside a BandMath. It contains : + - an ExtractROI app that handles extracting bands and ROI and can be written to disk or used in pipelines + - in case the user only wants to extract one band, an expression such as "im1b#" + + Args: + x: input + rows: rows slicing (e.g. 100:2000) + cols: columns slicing (e.g. 100:2000) + channels: channels, can be slicing, list or int + + """ + # Initialize the app that will be used for writing the slicer + self.output_parameter_key = 'out' + self.name = 'Slicer' + app = App('ExtractROI', {'in': x, 'mode': 'extent'}, propagate_pixel_type=True) + + parameters = {} + # Channel slicing + if channels != slice(None, None, None): + # Trigger source app execution if needed + nb_channels = get_nbchannels(x) + app.app.Execute() # this is needed by ExtractROI for setting the `cl` parameter + # if needed, converting int to list + if isinstance(channels, int): + channels = [channels] + # if needed, converting slice to list + elif isinstance(channels, slice): + channels_start = channels.start if channels.start is not None else 0 + channels_end = channels.stop if channels.stop is not None else nb_channels + channels_step = channels.step if channels.step is not None else 1 + channels = range(channels_start, channels_end, channels_step) + elif isinstance(channels, tuple): + channels = list(channels) + elif not isinstance(channels, list): + raise ValueError(f'Invalid type for channels, should be int, slice or list of bands. : {channels}') + + # Change the potential negative index values to reverse index + channels = [c if c >= 0 else nb_channels + c for c in channels] + parameters.update({'cl': [f'Channel{i + 1}' for i in channels]}) + + # Spatial slicing + spatial_slicing = False + # TODO: handle PixelValue app so that accessing value is possible, e.g. raster[120, 200, 0] + # TODO TBD: handle the step value in the slice so that NN undersampling is possible ? e.g. raster[::2, ::2] + if rows.start is not None: + parameters.update({'mode.extent.uly': rows.start}) + spatial_slicing = True + if rows.stop is not None and rows.stop != -1: + parameters.update( + {'mode.extent.lry': rows.stop - 1}) # subtract 1 to be compliant with python convention + spatial_slicing = True + if cols.start is not None: + parameters.update({'mode.extent.ulx': cols.start}) + spatial_slicing = True + if cols.stop is not None and cols.stop != -1: + parameters.update( + {'mode.extent.lrx': cols.stop - 1}) # subtract 1 to be compliant with python convention + spatial_slicing = True + # Execute app + app.set_parameters(**parameters) + + # Keeping the OTB app and the pyotb app + self.pyotb_app, self.app = app, app.app + + # These are some attributes when the user simply wants to extract *one* band to be used in an Operation + if not spatial_slicing and isinstance(channels, list) and len(channels) == 1: + self.one_band_sliced = channels[0] + 1 # OTB convention: channels start at 1 + self.input = x + + +class Input(otbObject): + """ + Class for transforming a filepath to pyOTB object + """ + + def __init__(self, filepath): + """ + Args: + filepath: raster file path + + """ + self.output_parameter_key = 'out' + self.filepath = filepath + self.name = f'Input from {filepath}' + app = App('ExtractROI', filepath, execute=True, propagate_pixel_type=True) + + # Keeping the OTB app and the pyotb app + self.pyotb_app, self.app = app, app.app + + def __str__(self): + """ + Returns: + string representation + + """ + return f'<pyotb.Input object from {self.filepath}>' + + +class Output(otbObject): + """ + Class for output of an app + """ + + def __init__(self, pyotb_app, output_parameter_key): + """ + Args: + app: The pyotb App + output_parameter_key: Output parameter key + + """ + # Keeping the OTB app and the pyotb app + self.pyotb_app, self.app = pyotb_app, pyotb_app.app + self.output_parameter_key = output_parameter_key + self.name = f'Output {output_parameter_key} from {self.app.GetName()}' + + def __str__(self): + """ + Returns: + string representation + + """ + return f'<pyotb.Output {self.app.GetName()} object, id {id(self)}>' + + class Operation(otbObject): """ Class for arithmetic/math operations done in Python. @@ -1056,7 +1048,7 @@ class Operation(otbObject): """ Given some inputs and an operator, this function enables to transform this into an OTB application. Operations generally involve 2 inputs (+, -...). It can have only 1 input for `abs` operator. - It can have 3 inputs for the ternary operator `cond ? x : y`, + It can have 3 inputs for the ternary operator `cond ? x : y`. Args: operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ? -- GitLab From 9c3d4fad8f569abfb601febf7a889894afd4158d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Nar=C3=A7on?= <nicolas.narcon@inrae.fr> Date: Mon, 4 Jul 2022 13:14:38 +0200 Subject: [PATCH 17/22] REFAC: revert contructor names to the initial state for backward comp --- pyotb/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index b13c7d4..fc45d14 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1001,7 +1001,7 @@ class Output(otbObject): Class for output of an app """ - def __init__(self, pyotb_app, output_parameter_key): + def __init__(self, app, output_parameter_key): """ Args: app: The pyotb App @@ -1009,7 +1009,7 @@ class Output(otbObject): """ # Keeping the OTB app and the pyotb app - self.pyotb_app, self.app = pyotb_app, pyotb_app.app + self.pyotb_app, self.app = app, app.app self.output_parameter_key = output_parameter_key self.name = f'Output {output_parameter_key} from {self.app.GetName()}' -- GitLab From a8cd4e07a959aad437e94b543540fb1e313b92a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Nar=C3=A7on?= <nicolas.narcon@inrae.fr> Date: Mon, 4 Jul 2022 13:22:53 +0200 Subject: [PATCH 18/22] FIX: do not set param when value is None --- pyotb/core.py | 55 ++++++++++++++++++++++++++------------------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index fc45d14..5ef8c6c 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -803,33 +803,34 @@ class App(otbObject): """ Set one parameter, decide which otb.Application method to use depending on target object """ - # Single-parameter cases - if isinstance(obj, otbObject): - self.app.ConnectImage(param, obj.app, obj.output_param) - elif isinstance(obj, otb.Application): # this is for backward comp with plain OTB - outparamkey = [param for param in obj.GetParametersKeys() - if obj.GetParameterType(param) == otb.ParameterType_OutputImage][0] - self.app.ConnectImage(param, obj, outparamkey) - elif param == 'ram': # SetParameterValue in OTB<7.4 doesn't work for ram parameter cf gitlab OTB issue 2200 - self.app.SetParameterInt('ram', int(obj)) - elif not isinstance(obj, list): # any other parameters (str, int...) - self.app.SetParameterValue(param, obj) - # Images list - elif self.__is_key_images_list(param): - # To enable possible in-memory connections, we go through the list and set the parameters one by one - for inp in obj: - if isinstance(inp, otbObject): - self.app.ConnectImage(param, inp.app, inp.output_param) - elif isinstance(inp, otb.Application): # this is for backward comp with plain OTB - outparamkey = [param for param in inp.GetParametersKeys() if - inp.GetParameterType(param) == otb.ParameterType_OutputImage][0] - self.app.ConnectImage(param, inp, outparamkey) - else: # here `input` should be an image filepath - # Append `input` to the list, do not overwrite any previously set element of the image list - self.app.AddParameterStringList(param, inp) - # List of any other types (str, int...) - else: - self.app.SetParameterValue(param, obj) + if obj is not None: + # Single-parameter cases + if isinstance(obj, otbObject): + self.app.ConnectImage(param, obj.app, obj.output_param) + elif isinstance(obj, otb.Application): # this is for backward comp with plain OTB + outparamkey = [param for param in obj.GetParametersKeys() + if obj.GetParameterType(param) == otb.ParameterType_OutputImage][0] + self.app.ConnectImage(param, obj, outparamkey) + elif param == 'ram': # SetParameterValue in OTB<7.4 doesn't work for ram parameter cf gitlab OTB issue 2200 + self.app.SetParameterInt('ram', int(obj)) + elif not isinstance(obj, list): # any other parameters (str, int...) + self.app.SetParameterValue(param, obj) + # Images list + elif self.__is_key_images_list(param): + # To enable possible in-memory connections, we go through the list and set the parameters one by one + for inp in obj: + if isinstance(inp, otbObject): + self.app.ConnectImage(param, inp.app, inp.output_param) + elif isinstance(inp, otb.Application): # this is for backward comp with plain OTB + outparamkey = [param for param in inp.GetParametersKeys() if + inp.GetParameterType(param) == otb.ParameterType_OutputImage][0] + self.app.ConnectImage(param, inp, outparamkey) + else: # here `input` should be an image filepath + # Append `input` to the list, do not overwrite any previously set element of the image list + self.app.AddParameterStringList(param, inp) + # List of any other types (str, int...) + else: + self.app.SetParameterValue(param, obj) def __propagate_pixel_type(self): """ -- GitLab From 87eb8dd7a6365d57f265ab9dabfdb24b3f5d90d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Nar=C3=A7on?= <nicolas.narcon@inrae.fr> Date: Mon, 4 Jul 2022 18:15:17 +0200 Subject: [PATCH 19/22] ENH: add an allow to failure for some tests --- tests/pipeline_test.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/pipeline_test.py b/tests/pipeline_test.py index 88c8212..1e02268 100644 --- a/tests/pipeline_test.py +++ b/tests/pipeline_test.py @@ -21,6 +21,10 @@ PYOTB_BLOCKS = [ ALL_BLOCKS = PYOTB_BLOCKS + OTBAPPS_BLOCKS +# These apps are problematic when used in pipelines with intermediate outputs +# (cf https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/2290) +PROBLEMATIC_APPS = ['DynamicConvert', 'BandMath'] + def backward(): """ @@ -175,10 +179,16 @@ for pipeline, errs in results.items(): msg = f"\033[91m{msg}\033[0m" msg += " | " if has_err: - nb_fails += 1 causes = [f"{section}: " + ", ".join([f"app{i}" for i in out_ids]) for section, out_ids in errs.items() if out_ids] msg += "\033[91mFAIL\033[0m (" + "; ".join(causes) + ")" + + # There is a failure when the pipeline length is >=3, the last app is an Operation and the first app of the + # piepline is one of the problematic apps + if ("write" in args and "backward" not in args and isinstance(pipeline[-1], pyotb.Operation) + and len(pipeline) == 3 and pipeline[0].name in PROBLEMATIC_APPS and pipeline[1].name not in PROBLEMATIC_APPS): + continue + nb_fails += 1 else: msg += "\033[92mPASS\033[0m" print(msg) -- GitLab From 807ddca251ca07c952a360e51687a9429eaaa727 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Nar=C3=A7on?= <nicolas.narcon@inrae.fr> Date: Mon, 4 Jul 2022 18:36:08 +0200 Subject: [PATCH 20/22] FIX: pipelines tests --- tests/pipeline_test.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/pipeline_test.py b/tests/pipeline_test.py index 1e02268..a3d7ff4 100644 --- a/tests/pipeline_test.py +++ b/tests/pipeline_test.py @@ -171,6 +171,7 @@ print(f'Tests summary (\033[93mTest options: {"; ".join(args)}\033[0m)') print("Pipeline".ljust(cols) + " | Status (reason)") print("-" * cols + "-|-" + "-" * 20) nb_fails = 0 +allowed_to_fail = 0 for pipeline, errs in results.items(): has_err = sum(len(value) for key, value in errs.items()) > 0 graph = pipeline2str(pipeline) @@ -186,11 +187,12 @@ for pipeline, errs in results.items(): # There is a failure when the pipeline length is >=3, the last app is an Operation and the first app of the # piepline is one of the problematic apps if ("write" in args and "backward" not in args and isinstance(pipeline[-1], pyotb.Operation) - and len(pipeline) == 3 and pipeline[0].name in PROBLEMATIC_APPS and pipeline[1].name not in PROBLEMATIC_APPS): - continue - nb_fails += 1 + and len(pipeline) == 3 and pipeline[0].name in PROBLEMATIC_APPS): + allowed_to_fail += 1 + else: + nb_fails += 1 else: msg += "\033[92mPASS\033[0m" print(msg) -print(f"End of summary ({nb_fails} error(s)).", flush=True) +print(f"End of summary ({nb_fails} error(s), {allowed_to_fail} 'allowed to fail' error(s))", flush=True) assert nb_fails == 0, "One of the pipelines have failed. Please read the report." -- GitLab From 2d9195a3f9d25e8d74b50e535caeb3ec218d8a06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Nar=C3=A7on?= <nicolas.narcon@inrae.fr> Date: Tue, 5 Jul 2022 07:38:59 +0200 Subject: [PATCH 21/22] ENH: remove commented piece of code --- pyotb/core.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 5ef8c6c..791f088 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -606,10 +606,6 @@ class App(otbObject): dtypes = {key: parse_pixel_type(pixel_type) for key in self.output_parameters_keys} for key, typ in dtypes.items(): self.app.SetParameterOutputImagePixelType(key, typ) - # Here we make sure that intermediate outputs will be flushed to disk - if self.__with_output(): - pass - # self.app.PropagateConnectMode(True) # Run app, write output if needed, update `finished` property if execute or not self.output_param: self.execute() -- GitLab From aa8ea2c5b96c95d8408a71d6ec88df699653ba3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Nar=C3=A7on?= <nicolas.narcon@inrae.fr> Date: Tue, 5 Jul 2022 07:47:17 +0200 Subject: [PATCH 22/22] STYLE: bump version --- README.md | 2 +- doc/installation.md | 2 +- doc/troubleshooting.md | 4 +++- pyotb/__init__.py | 2 +- setup.py | 2 +- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9879d45..6f0adcd 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Requirements: pip install pyotb --upgrade ``` -For Python>=3.6, latest version available is pyotb 1.4.0 For Python 3.5, latest version available is pyotb 1.2.2 +For Python>=3.6, latest version available is pyotb 1.4.1 For Python 3.5, latest version available is pyotb 1.2.2 ## Quickstart: running an OTB application as a oneliner pyotb has been written so that it is more convenient to run an application in Python. diff --git a/doc/installation.md b/doc/installation.md index 991ecb9..6c79ef8 100644 --- a/doc/installation.md +++ b/doc/installation.md @@ -8,4 +8,4 @@ pip install pyotb --upgrade ``` -For Python>=3.6, latest version available is pyotb 1.4.0. For Python 3.5, latest version available is pyotb 1.2.2 +For Python>=3.6, latest version available is pyotb 1.4.1. For Python 3.5, latest version available is pyotb 1.2.2 diff --git a/doc/troubleshooting.md b/doc/troubleshooting.md index aae8be6..e6b0b4c 100644 --- a/doc/troubleshooting.md +++ b/doc/troubleshooting.md @@ -54,7 +54,7 @@ one_band.write('one_band.tif') #### Example of failures involving arithmetic operation -One can meet errors when using arithmetic operations raster at the end of a pipeline when DynamicConvert, BandMath or +One can meet errors when using arithmetic operations at the end of a pipeline when DynamicConvert, BandMath or OpticalCalibration is involved: ```python @@ -69,3 +69,5 @@ inp.write('one_band.tif') inp_new.write('one_band_nodata.tif') # Failure here absolute.write('absolute.tif') # Failure here ``` + +When writing only the final result, i.e. the end of the pipeline (`absolute.write('absolute.tif')`), there is no problem. diff --git a/pyotb/__init__.py b/pyotb/__init__.py index a57a623..561ef64 100644 --- a/pyotb/__init__.py +++ b/pyotb/__init__.py @@ -2,7 +2,7 @@ """ This module provides convenient python wrapping of otbApplications """ -__version__ = "1.4.0" +__version__ = "1.4.1" from .apps import * from .core import App, Output, Input, get_nbchannels, get_pixel_type diff --git a/setup.py b/setup.py index 7785ddc..c99a422 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding="utf-8") as fh: setuptools.setup( name="pyotb", - version="1.4.0", + version="1.4.1", author="Nicolas Narçon", author_email="nicolas.narcon@gmail.com", description="Library to enable easy use of the Orfeo Tool Box (OTB) in Python", -- GitLab