Processing/processing tools sampler test

.github/workflows/tester.yml

@@ -2,16 +2,12 @@ name: "🎳 Tester"
 
 on:
   push:
-    branches:
-      - main
     paths:
       - '**.py'
       - .github/workflows/tester.yml
       - requirements/testing.txt
 
   pull_request:
-    branches:
-      - main
     paths:
       - '**.py'
       - .github/workflows/tester.yml
@@ -45,55 +41,62 @@ jobs:
       - name: Run Unit tests
         run: pytest -p no:qgis tests/unit/
 
-  # test-qgis:
-  #   runs-on: ubuntu-latest
-
-  #   container:
-  #     image: qgis/qgis:3.4
-  #     env:
-  #       CI: true
-  #       DISPLAY: ":1"
-  #       MUTE_LOGS: true
-  #       NO_MODALS: 1
-  #       PYTHONPATH: "/usr/share/qgis/python/plugins:/usr/share/qgis/python:."
-  #       QT_QPA_PLATFORM: "offscreen"
-  #       WITH_PYTHON_PEP: false
-  #     # be careful, things have changed since QGIS 3.40. So if you are using this setup 
-  #     #  with a QGIS version older than 3.40, you may need to change the way you set up the container
-  #     volumes:
-  #       # Mount the X11 socket to allow GUI applications to run
-  #       - /tmp/.X11-unix:/tmp/.X11-unix
-  #       # Mount the workspace directory to the container
-  #       - ${{ github.workspace }}:/home/root/
-
-  #   steps:
-  #     - name: Get source code
-  #       uses: actions/checkout@v4
-
-  #     - name: Print QGIS version
-  #       run: qgis --version
-
-  #     # Uncomment if you need to run a script to set up the plugin in QGIS docker image < 3.40
-  #     # - name: Setup plugin
-  #     #   run: qgis_setup.sh ${{ env.PROJECT_FOLDER }}
-
-  #     - name: Install Python requirements
-  #       run: |
-  #         apt update && apt install -y python3-pip python3-venv pipx
-  #         # Create a virtual environment
-  #         cd /home/root/
-  #         pipx run qgis-venv-creator --venv-name ".venv"
-  #         # Activate the virtual environment
-  #         . .venv/bin/activate
-  #         # Install the requirements
-  #         python3 -m pip install -U -r requirements/testing.txt
-
-  #     - name: Run Unit tests
-  #       run: |
-  #         cd /home/root/
-  #         # Activate the virtual environment
-  #         . .venv/bin/activate
-  #         # Run the tests
-  #         # xvfb-run is used to run the tests in a virtual framebuffer
-  #         # This is necessary because QGIS requires a display to run
-  #         xvfb-run python3 -m pytest tests/qgis --junitxml=junit/test-results-qgis.xml --cov-report=xml:coverage-reports/coverage-qgis.xml
+  test-qgis:
+    runs-on: ubuntu-latest
+
+    container:
+      image: qgis/qgis:latest
+      env:
+        CI: true
+        DISPLAY: ":1"
+        MUTE_LOGS: true
+        NO_MODALS: 1
+        PYTHONPATH: "/usr/share/qgis/python/plugins:/usr/share/qgis/python:."
+        QT_QPA_PLATFORM: "offscreen"
+        WITH_PYTHON_PEP: false
+      # be careful, things have changed since QGIS 3.40. So if you are using this setup 
+      #  with a QGIS version older than 3.40, you may need to change the way you set up the container
+      volumes:
+        # Mount the X11 socket to allow GUI applications to run
+        - /tmp/.X11-unix:/tmp/.X11-unix
+        # Mount the workspace directory to the container
+        - ${{ github.workspace }}:/home/root/
+
+    steps:
+      - name: Get source code
+        uses: actions/checkout@v4
+
+      - name: Print QGIS version
+        run: qgis --version
+
+      # Uncomment if you need to run a script to set up the plugin in QGIS docker image < 3.40
+      # - name: Setup plugin
+      #   run: qgis_setup.sh ${{ env.PROJECT_FOLDER }}
+
+      - name: Install Python requirements
+        run: |
+          apt update && apt install -y python3-pip python3-venv pipx
+          # Create a virtual environment
+          cd /home/root/
+          pipx run qgis-venv-creator --venv-name ".venv"
+          # Activate the virtual environment
+          . .venv/bin/activate
+          # Install the requirements
+          python3 -m pip install -U -r requirements/testing.txt
+          python3 -m pip install git+https://github.com/Loop3D/map2loop.git@noelle/contact_extractor
+
+      - name: verify input data
+        run: |
+          cd /home/root/
+          . .venv/bin/activate
+          ls -la tests/qgis/input/ || echo "Input directory not found"
+
+      - name: Run Unit tests
+        run: |
+          cd /home/root/
+          # Activate the virtual environment
+          . .venv/bin/activate
+          # Run the tests
+          # xvfb-run is used to run the tests in a virtual framebuffer
+          # This is necessary because QGIS requires a display to run
+          xvfb-run python3 -m pytest tests/qgis --junitxml=junit/test-results-qgis.xml --cov-report=xml:coverage-reports/coverage-qgis.xml

m2l/processing/algorithms/__init__.py

@@ -1,4 +1,4 @@
 from .extract_basal_contacts import BasalContactsAlgorithm
 from .sorter import StratigraphySorterAlgorithm
 from .thickness_calculator import ThicknessCalculatorAlgorithm
-from .sampler import SamplerAlgorithm
+from .sampler import SamplerAlgorithm

m2l/processing/algorithms/sampler.py

@@ -11,6 +11,8 @@
 # Python imports
 from typing import Any, Optional
 from qgis.PyQt.QtCore import QMetaType
+from osgeo import gdal
+import pandas as pd
 
 # QGIS imports
 from qgis.core import (
@@ -22,13 +24,17 @@
     QgsProcessingFeedback,
     QgsProcessingParameterFeatureSink,
     QgsProcessingParameterFeatureSource,
-    QgsProcessingParameterString,
+    QgsProcessingParameterRasterLayer,
+    QgsProcessingParameterEnum,
     QgsProcessingParameterNumber,
+    QgsFields,
     QgsField,
     QgsFeature,
     QgsGeometry,
     QgsPointXY,
-    QgsVectorLayer
+    QgsVectorLayer,
+    QgsWkbTypes,
+    QgsCoordinateReferenceSystem
 )
 # Internal imports
 from ...main.vectorLayerWrapper import qgsLayerToGeoDataFrame
@@ -68,17 +74,20 @@ def initAlgorithm(self, config: Optional[dict[str, Any]] = None) -> None:
 
 
         self.addParameter(
-            QgsProcessingParameterString(
+            QgsProcessingParameterEnum(
                 self.INPUT_SAMPLER_TYPE,
                 "SAMPLER_TYPE",
+                ["Decimator", "Spacing"],
+                defaultValue=0
             )
         )
 
         self.addParameter(
-            QgsProcessingParameterFeatureSource(
+            QgsProcessingParameterRasterLayer(
                 self.INPUT_DTM,
                 "DTM",
                 [QgsProcessing.TypeRaster],
+                optional=True,
             )
         )
 
@@ -104,6 +113,8 @@ def initAlgorithm(self, config: Optional[dict[str, Any]] = None) -> None:
             QgsProcessingParameterNumber(
                 self.INPUT_DECIMATION,
                 "DECIMATION",
+                QgsProcessingParameterNumber.Integer,
+                defaultValue=1,
                 optional=True,
             )
         )
@@ -112,14 +123,16 @@ def initAlgorithm(self, config: Optional[dict[str, Any]] = None) -> None:
             QgsProcessingParameterNumber(
                 self.INPUT_SPACING,
                 "SPACING",
+                QgsProcessingParameterNumber.Double,
+                defaultValue=200.0,
                 optional=True,
             )
         )
 
         self.addParameter(
             QgsProcessingParameterFeatureSink(
                 self.OUTPUT,
-                "Sampled Contacts",
+                "Sampled Points",
             )
         )
 
@@ -130,60 +143,80 @@ def processAlgorithm(
         feedback: QgsProcessingFeedback,
     ) -> dict[str, Any]:
 
-        dtm = self.parameterAsSource(parameters, self.INPUT_DTM, context)
-        geology = self.parameterAsSource(parameters, self.INPUT_GEOLOGY, context)
-        spatial_data = self.parameterAsSource(parameters, self.INPUT_SPATIAL_DATA, context)
-        decimation = self.parameterAsSource(parameters, self.INPUT_DECIMATION, context)
-        spacing = self.parameterAsSource(parameters, self.INPUT_SPACING, context)
-        sampler_type = self.parameterAsString(parameters, self.INPUT_SAMPLER_TYPE, context)
+        dtm = self.parameterAsRasterLayer(parameters, self.INPUT_DTM, context)
+        geology = self.parameterAsVectorLayer(parameters, self.INPUT_GEOLOGY, context)
+        spatial_data = self.parameterAsVectorLayer(parameters, self.INPUT_SPATIAL_DATA, context)
+        decimation = self.parameterAsInt(parameters, self.INPUT_DECIMATION, context)
+        spacing = self.parameterAsDouble(parameters, self.INPUT_SPACING, context)
+        sampler_type_index = self.parameterAsEnum(parameters, self.INPUT_SAMPLER_TYPE, context)
+        sampler_type = ["Decimator", "Spacing"][sampler_type_index]
+
+        if spatial_data is None:
+            raise QgsProcessingException("Spatial data is required")
+
+        if sampler_type is "Decimator":
+            if geology is None:
+                raise QgsProcessingException("Geology is required")
+            if dtm is None:
+                raise QgsProcessingException("DTM is required")
 
         # Convert geology layers to GeoDataFrames
         geology = qgsLayerToGeoDataFrame(geology)
-        spatial_data = qgsLayerToGeoDataFrame(spatial_data)
+        spatial_data_gdf = qgsLayerToGeoDataFrame(spatial_data)
+        dtm_gdal = gdal.Open(dtm.source()) if dtm is not None and dtm.isValid() else None
 
-        if sampler_type == "decimator":
+        if sampler_type == "Decimator":
             feedback.pushInfo("Sampling...")
-            sampler = SamplerDecimator(decimation=decimation, dtm_data=dtm, geology_data=geology, feedback=feedback)
-            samples = sampler.sample(spatial_data)
+            sampler = SamplerDecimator(decimation=decimation, dtm_data=dtm_gdal, geology_data=geology)
+            samples = sampler.sample(spatial_data_gdf)
 
-        if sampler_type == "spacing":
+        if sampler_type == "Spacing":
             feedback.pushInfo("Sampling...")
-            sampler = SamplerSpacing(spacing=spacing, dtm_data=dtm, geology_data=geology, feedback=feedback)
-            samples = sampler.sample(spatial_data)
-
-
-        # create layer
-        vector_layer = QgsVectorLayer("Point", "sampled_points", "memory")
-        provider = vector_layer.dataProvider()
-
-        # add fields
-        provider.addAttributes([QgsField("ID", QMetaType.Type.QString),
-                            QgsField("X",  QMetaType.Type.Float),
-                            QgsField("Y",  QMetaType.Type.Float),
-                            QgsField("Z",  QMetaType.Type.Float),
-                            QgsField("featureId", QMetaType.Type.QString)
-                            ])
-        vector_layer.updateFields() # tell the vector layer to fetch changes from the provider
-
-        # add a feature
-        for i in range(len(samples)):
-            feature = QgsFeature()
-            feature.setGeometry(QgsGeometry.fromPointXY(QgsPointXY(samples.X[i], samples.Y[i], samples.Z[i])))
-            feature.setAttributes([samples.ID[i], samples.X[i], samples.Y[i], samples.Z[i], samples.featureId[i]])
-            provider.addFeatures([feature])
-
-        # update layer's extent when new features have been added
-        # because change of extent in provider is not propagated to the layer
-        vector_layer.updateExtents()
-        # --- create sink
+            sampler = SamplerSpacing(spacing=spacing, dtm_data=dtm_gdal, geology_data=geology)
+            samples = sampler.sample(spatial_data_gdf)
+
+        fields = QgsFields()
+        fields.append(QgsField("ID", QMetaType.Type.QString))
+        fields.append(QgsField("X", QMetaType.Type.Float))
+        fields.append(QgsField("Y", QMetaType.Type.Float))
+        fields.append(QgsField("Z", QMetaType.Type.Float))
+        fields.append(QgsField("featureId", QMetaType.Type.QString))
+
+        crs = None
+        if spatial_data_gdf is not None and spatial_data_gdf.crs is not None:
+            crs = QgsCoordinateReferenceSystem.fromWkt(spatial_data_gdf.crs.to_wkt())
+
         sink, dest_id = self.parameterAsSink(
             parameters,
             self.OUTPUT,
             context,
-            vector_layer.fields(),
-            QgsGeometry.Type.Point,
-            spatial_data.crs,
-            )
+            fields,
+            QgsWkbTypes.PointZ if 'Z' in (samples.columns if samples is not None else []) else QgsWkbTypes.Point,
+            crs
+        )
+
+        if samples is not None and not samples.empty:
+            for _index, row in samples.iterrows():
+                feature = QgsFeature(fields)
+
+                # decimator has z values
+                if 'Z' in samples.columns and pd.notna(row.get('Z')):
+                    wkt = f"POINT Z ({row['X']} {row['Y']} {row['Z']})"
+                    feature.setGeometry(QgsGeometry.fromWkt(wkt))
+                else:
+                    #spacing has no z values
+                    feature.setGeometry(QgsGeometry.fromPointXY(QgsPointXY(row['X'], row['Y'])))
+
+                feature.setAttributes([
+                    str(row.get('ID', '')),
+                    float(row.get('X', 0)),
+                    float(row.get('Y', 0)),
+                    float(row.get('Z', 0)) if pd.notna(row.get('Z')) else 0.0,
+                    str(row.get('featureId', ''))
+                ])
+
+                sink.addFeature(feature)
+
         return {self.OUTPUT: dest_id}
 
     def createInstance(self) -> QgsProcessingAlgorithm:

m2l/processing/algorithms/sorter.py

@@ -92,7 +92,7 @@ def initAlgorithm(self, config: Optional[dict[str, Any]] = None) -> None:
         self.addParameter(
             QgsProcessingParameterFeatureSink(
                 self.OUTPUT,
-                self.tr("Stratigraphic column"),
+                "Stratigraphic column",
             )
         )
 
@@ -177,7 +177,7 @@ def build_input_frames(layer: QgsVectorLayer, feedback) -> tuple:
     (units_df, relationships_df, contacts_df, map_data)
     """
     import pandas as pd
-    from m2l.map2loop.mapdata import MapData  # adjust import path if needed
+    from map2loop.map2loop.mapdata import MapData  # adjust import path if needed
 
     # Example: convert the geology layer to a very small units_df
     units_records = []

requirements/testing.txt

@@ -3,3 +3,5 @@
 
 pytest-cov>=4
 packaging>=23
+shapely
+geopandas

tests/qgis/input/dtm_rp.tif.aux.xml

@@ -0,0 +1,11 @@
+<PAMDataset>
+  <PAMRasterBand band="1">
+    <Metadata>
+      <MDI key="STATISTICS_MAXIMUM">1175</MDI>
+      <MDI key="STATISTICS_MEAN">561.72162965205</MDI>
+      <MDI key="STATISTICS_MINIMUM">297</MDI>
+      <MDI key="STATISTICS_STDDEV">107.52060579962</MDI>
+      <MDI key="STATISTICS_VALID_PERCENT">99.73</MDI>
+    </Metadata>
+  </PAMRasterBand>
+</PAMDataset>

tests/qgis/input/faults_clip.cpg

@@ -0,0 +1 @@
+ISO-8859-1

tests/qgis/input/faults_clip.prj

@@ -0,0 +1 @@
+PROJCS["GDA94_MGA_zone_50",GEOGCS["GCS_GDA94",DATUM["D_Geocentric_Datum_of_Australia_1994",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Transverse_Mercator"],PARAMETER["False_Easting",500000.0],PARAMETER["False_Northing",10000000.0],PARAMETER["Central_Meridian",117.0],PARAMETER["Scale_Factor",0.9996],PARAMETER["Latitude_Of_Origin",0.0],UNIT["Meter",1.0]]

tests/qgis/input/folds_clip.cpg

@@ -0,0 +1 @@
+ISO-8859-1

tests/qgis/input/folds_clip.prj

@@ -0,0 +1 @@
+PROJCS["GDA94_MGA_zone_50",GEOGCS["GCS_GDA94",DATUM["D_Geocentric_Datum_of_Australia_1994",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Transverse_Mercator"],PARAMETER["False_Easting",500000.0],PARAMETER["False_Northing",10000000.0],PARAMETER["Central_Meridian",117.0],PARAMETER["Scale_Factor",0.9996],PARAMETER["Latitude_Of_Origin",0.0],UNIT["Meter",1.0]]

tests/qgis/input/geol_clip_no_gaps.cpg

@@ -0,0 +1 @@
+ISO-8859-1

tests/qgis/input/geol_clip_no_gaps.prj

@@ -0,0 +1 @@
+PROJCS["GDA94_MGA_zone_50",GEOGCS["GCS_GDA94",DATUM["D_Geocentric_Datum_of_Australia_1994",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Transverse_Mercator"],PARAMETER["False_Easting",500000.0],PARAMETER["False_Northing",10000000.0],PARAMETER["Central_Meridian",117.0],PARAMETER["Scale_Factor",0.9996],PARAMETER["Latitude_Of_Origin",0.0],UNIT["Meter",1.0]]

tests/qgis/input/structure_clip.cpg

@@ -0,0 +1 @@
+ISO-8859-1