From f8d9b28d16a074b3756f291503e37f09758abfe1 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Dec 2025 13:08:19 +0000 Subject: [PATCH 01/22] switch from property to method --- activestorage/active.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/activestorage/active.py b/activestorage/active.py index 7ffe141c..a7da2277 100644 --- a/activestorage/active.py +++ b/activestorage/active.py @@ -362,19 +362,25 @@ def method(self, value): self._method = value - @property - def mean(self): + + def mean(self, axis=None): self._method = "mean" + if axis is not None: + self.axis = axis return self - @property - def min(self): + + def min(self, axis=None): self._method = "min" + if axis is not None: + self.axis = axis return self - @property - def max(self): + + def max(self, axis=None): self._method = "max" + if axis is not None: + self.axis = axis return self @property From 59d54b3843c418b81a77cff2a0529ac6ed296988 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Dec 2025 13:08:45 +0000 Subject: [PATCH 02/22] start fixing tests --- tests/test_bigger_data.py | 12 ++++++------ tests/unit/test_active_axis.py | 23 +++++++++++++++++++++-- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/tests/test_bigger_data.py b/tests/test_bigger_data.py index a305dec1..99446489 100644 --- a/tests/test_bigger_data.py +++ b/tests/test_bigger_data.py @@ -136,7 +136,7 @@ def test_cl_mean(tmp_path): active = Active(ncfile, "cl", storage_type=utils.get_storage_type()) active._version = 2 active.components = True - result2 = active.mean[4:5, 1:2] + result2 = active.mean()[4:5, 1:2] print(result2, ncfile) # expect {'sum': array([[[[264.]]]], dtype=float32), 'n': array([[[[12]]]])} # check for typing and structure @@ -151,7 +151,7 @@ def test_cl_min(tmp_path): ncfile = save_cl_file_with_a(tmp_path) active = Active(ncfile, "cl", storage_type=utils.get_storage_type()) active._version = 2 - result2 = active.min[4:5, 1:2] + result2 = active.min()[4:5, 1:2] np.testing.assert_array_equal(result2, np.array([[[[22.]]]], dtype="float32")) @@ -160,7 +160,7 @@ def test_cl_max(tmp_path): ncfile = save_cl_file_with_a(tmp_path) active = Active(ncfile, "cl", storage_type=utils.get_storage_type()) active._version = 2 - result2 = active.max[4:5, 1:2] + result2 = active.max()[4:5, 1:2] np.testing.assert_array_equal(result2, np.array([[[[22.]]]], dtype="float32")) @@ -169,7 +169,7 @@ def test_cl_global_max(tmp_path): ncfile = save_cl_file_with_a(tmp_path) active = Active(ncfile, "cl", storage_type=utils.get_storage_type()) active._version = 2 - result2 = active.max[:] + result2 = active.max()[:] np.testing.assert_array_equal(result2, np.array([[[[22.]]]], dtype="float32")) @@ -192,7 +192,7 @@ def test_ps(tmp_path): active = Active(ncfile, "ps", storage_type=utils.get_storage_type()) active._version = 2 active.components = True - result2 = active.mean[4:5, 1:2] + result2 = active.mean()[4:5, 1:2] print(result2, ncfile) # expect {'sum': array([[[22.]]]), 'n': array([[[4]]])} # check for typing and structure @@ -381,7 +381,7 @@ def test_daily_data_masked_two_stats(test_data_path): # first a mean active = Active(uri, "ta", storage_type=utils.get_storage_type()) active._version = 2 - result2 = active.min[:] + result2 = active.min()[:] assert result2 == 245.0020751953125 # then recycle Active object for something else diff --git a/tests/unit/test_active_axis.py b/tests/unit/test_active_axis.py index 3bbc085f..05712394 100644 --- a/tests/unit/test_active_axis.py +++ b/tests/unit/test_active_axis.py @@ -76,14 +76,33 @@ def test_active_axis_format_1(): active1 = Active(rfile, ncvar, axis=[0, 2]) active2 = Active(rfile, ncvar, axis=(-1, -3)) - x1 = active2.mean[...] - x2 = active2.mean[...] + x1 = active2.mean()[...] + x2 = active2.mean()[...] assert x1.shape == x2.shape assert (x1.mask == x2.mask).all() assert np.ma.allclose(x1, x2) +def test_active_axis_format_new_api(): + """Unit test for class:Active axis format with Numpy-style API.""" + active1 = Active(rfile, ncvar) + active2 = Active(rfile, ncvar) + + x1 = active2.mean(axis=[0, 2])[...] + assert active2.axis == [0, 2] + x2 = active2.mean(axis=(-1, -3))[...] + assert active2.axis == (-1, -3) + + assert x1.shape == x2.shape + assert (x1.mask == x2.mask).all() + assert np.ma.allclose(x1, x2) + + xmin = active2.min(axis=[0, 2])[...] + xmax = active2.min(axis=[0, 2])[...] + assert xmin == xmax == [[[198.82859802246094]]] + + def test_active_axis_format_2(): """Unit test for class:Active axis format.""" # Disallow out-of-range axes From 554fe6fd88d38b13ac6c275e94ca0b46d2115b77 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Dec 2025 13:12:16 +0000 Subject: [PATCH 03/22] more test fixing --- tests/test_real_https.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_real_https.py b/tests/test_real_https.py index 5a718c35..2bb6f7cf 100644 --- a/tests/test_real_https.py +++ b/tests/test_real_https.py @@ -15,7 +15,7 @@ def test_https(): active = Active(test_file_uri, "cl", storage_type="https") active._version = 1 - result = active.min[0:3, 4:6, 7:9] + result = active.min()[0:3, 4:6, 7:9] print("Result is", result) assert result == np.array([0.6909787], dtype="float32") @@ -26,7 +26,7 @@ def test_https_100years(): test_file_uri = "https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/CMIP/MOHC/UKESM1-1-LL/historical/r1i1p1f2/Amon/pr/gn/latest/pr_Amon_UKESM1-1-LL_historical_r1i1p1f2_gn_195001-201412.nc" active = Active(test_file_uri, "pr") active._version = 1 - result = active.min[0:3, 4:6, 7:9] + result = active.min()[0:3, 4:6, 7:9] print("Result is", result) assert result == np.array([5.4734613e-07], dtype="float32") @@ -43,7 +43,7 @@ def test_https_reductionist(): with pytest.raises(activestorage.reductionist.ReductionistError): active = Active(test_file_uri, "cl") active._version = 2 - result = active.min[0:3, 4:6, 7:9] + result = active.min()[0:3, 4:6, 7:9] print("Result is", result) assert result == np.array([0.6909787], dtype="float32") @@ -57,7 +57,7 @@ def test_https_implicit_storage(): active = Active(test_file_uri, "cl") active._version = 1 - result = active.min[0:3, 4:6, 7:9] + result = active.min()[0:3, 4:6, 7:9] print("Result is", result) assert result == np.array([0.6909787], dtype="float32") @@ -73,7 +73,7 @@ def test_https_implicit_storage_file_not_found(): with pytest.raises(FileNotFoundError): active = Active(test_file_uri, "cl") active._version = 1 - result = active.min[0:3, 4:6, 7:9] + result = active.min()[0:3, 4:6, 7:9] def test_https_implicit_storage_wrong_url(): @@ -98,7 +98,7 @@ def test_https_dataset(): active = Active(av, storage_type="https") active._version = 1 - result = active.min[0:3, 4:6, 7:9] + result = active.min()[0:3, 4:6, 7:9] print("Result is", result) assert result == np.array([0.6909787], dtype="float32") @@ -114,6 +114,6 @@ def test_https_dataset_implicit_storage(): active = Active(av) active._version = 1 - result = active.min[0:3, 4:6, 7:9] + result = active.min()[0:3, 4:6, 7:9] print("Result is", result) assert result == np.array([0.6909787], dtype="float32") From 8c82b698f8fc45498b879b9d6a3641536e8b1c78 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Dec 2025 13:14:07 +0000 Subject: [PATCH 04/22] more test fixing --- tests/unit/test_active.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_active.py b/tests/unit/test_active.py index 30453e5b..9c30b12f 100644 --- a/tests/unit/test_active.py +++ b/tests/unit/test_active.py @@ -112,7 +112,7 @@ def test_activevariable_pyfive_with_attributed_min(): ncvar = "TREFHT" ds = pyfive.File(uri)[ncvar] av = Active(ds) - av_slice_min = av.min[3:5] + av_slice_min = av.min()[3:5] assert av_slice_min == np.array(258.62814, dtype="float32") # test with Numpy np_slice_min = np.min(ds[3:5]) @@ -125,7 +125,7 @@ def test_activevariable_pyfive_with_attributed_mean(): ds = pyfive.File(uri)[ncvar] av = Active(ds) av.components = True - av_slice_min = av.mean[3:5] + av_slice_min = av.mean()[3:5] actual_mean = av_slice_min["sum"] / av_slice_min["n"] assert actual_mean == np.array(283.39508056640625, dtype="float32") # test with Numpy From 934b01380e52e06071cc59f0b42d0f9f548a8d03 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Dec 2025 13:15:47 +0000 Subject: [PATCH 05/22] final test fixing --- tests/test_real_s3.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_real_s3.py b/tests/test_real_s3.py index 70e774f1..8821a9a7 100644 --- a/tests/test_real_s3.py +++ b/tests/test_real_s3.py @@ -39,7 +39,7 @@ def test_s3_dataset(): storage_options=storage_options, active_storage_url=active_storage_url) active._version = 2 - result = active.min[0:3, 4:6, 7:9] # standardized slice + result = active.min()[0:3, 4:6, 7:9] # standardized slice print("Result is", result) assert result == 5098.625 @@ -49,7 +49,7 @@ def test_s3_dataset(): storage_options=storage_options, active_storage_url=active_storage_url) active._version = 2 - result = active.min[0:3, 4:6, 7:9] # standardized slice + result = active.min()[0:3, 4:6, 7:9] # standardized slice print("Result is", result) assert result == 5098.625 @@ -63,7 +63,7 @@ def test_s3_dataset(): storage_options=storage_options, active_storage_url=active_storage_url) active._version = 2 - result = active.min[0:3, 4:6, 7:9] # standardized slice + result = active.min()[0:3, 4:6, 7:9] # standardized slice print("Result is", result) assert result == 5098.625 @@ -72,6 +72,6 @@ def test_s3_dataset(): storage_options=storage_options, active_storage_url=active_storage_url) active._version = 2 - result = active.min[0:3, 4:6, 7:9] # standardized slice + result = active.min()[0:3, 4:6, 7:9] # standardized slice print("Result is", result) assert result == 5098.625 From 1f47bcbdd6c648610996cca43a67f55429a2717c Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Dec 2025 13:33:41 +0000 Subject: [PATCH 06/22] real s3 axis test --- tests/test_real_s3_with_axes.py | 54 +++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 tests/test_real_s3_with_axes.py diff --git a/tests/test_real_s3_with_axes.py b/tests/test_real_s3_with_axes.py new file mode 100644 index 00000000..b81c221f --- /dev/null +++ b/tests/test_real_s3_with_axes.py @@ -0,0 +1,54 @@ +import os +import numpy as np + +from activestorage.active import Active + + +S3_BUCKET = "bnl" + +def build_active(): + """Run an integration test with real data off S3.""" + storage_options = { + 'key': "f2d55c6dcfc7618b2c34e00b58df3cef", + 'secret': "$/'#M{0{/4rVhp%n^(XeX$q@y#&(NM3W1->~N.Q6VP.5[@bLpi='nt]AfH)>78pT", + 'client_kwargs': {'endpoint_url': "https://uor-aces-o.s3-ext.jc.rl.ac.uk"}, # final proxy + } + active_storage_url = "https://reductionist.jasmin.ac.uk/" # Wacasoft new Reductionist + bigger_file = "da193a_25_6hr_t_pt_cordex__198807-198807.nc" # m01s30i111 ## older 3GB 30 chunks + + test_file_uri = os.path.join( + S3_BUCKET, + bigger_file + ) + print("S3 Test file path:", test_file_uri) + active = Active(test_file_uri, 'm01s30i111', storage_type="s3", # 'm01s06i247_4', storage_type="s3", + storage_options=storage_options, + active_storage_url=active_storage_url) + + active._version = 2 + + return active + + +def test_no_axis(): + active = build_active() + result = active.min()[:] + assert result == [[[[164.8125]]]] + + +def test_no_axis_2(): + active = build_active() + result = active.min(axis=())[:] + assert result == [[[[164.8125]]]] + + +def test_axis_0_1(): + active = build_active() + result = active.min(axis=(0, 1))[:] + assert result == [[[[164.8125]]]] + + +def test_no_axis_1(): + active = build_active() + result = active.min(axis=(1))[:] + assert result == [[[[164.8125]]]] From b88a55251896135bc8a0dd1f2b672061c2ffd6d7 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Dec 2025 13:33:52 +0000 Subject: [PATCH 07/22] turn on axis --- activestorage/reductionist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activestorage/reductionist.py b/activestorage/reductionist.py index 66240dd5..4cc4089d 100644 --- a/activestorage/reductionist.py +++ b/activestorage/reductionist.py @@ -10,7 +10,7 @@ import numpy as np import requests -REDUCTIONIST_AXIS_READY = False +REDUCTIONIST_AXIS_READY = True DEBUG = 0 From 23acfde900c13cf47c56c66078674065df832afb Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Dec 2025 13:50:22 +0000 Subject: [PATCH 08/22] add axis to mock expected --- tests/unit/test_reductionist.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unit/test_reductionist.py b/tests/unit/test_reductionist.py index 4be797db..3b06670e 100644 --- a/tests/unit/test_reductionist.py +++ b/tests/unit/test_reductionist.py @@ -149,6 +149,8 @@ def test_reduce_chunk_compression(mock_request, compression, filters): "id": filter.codec_id, "element_size": filter.elementsize } for filter in filters], + "axis": + axis, } mock_request.assert_called_once_with(session, expected_url, expected_data) @@ -258,6 +260,8 @@ def test_reduce_chunk_missing(mock_request, missing): ]], "missing": api_arg, + "axis": + axis, } mock_request.assert_called_once_with(session, expected_url, expected_data) From 29fcaee95e2e0a530761a6d424b6db704493686b Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Dec 2025 14:40:39 +0000 Subject: [PATCH 09/22] fix bug --- activestorage/active.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/activestorage/active.py b/activestorage/active.py index a7da2277..220212b7 100644 --- a/activestorage/active.py +++ b/activestorage/active.py @@ -272,6 +272,7 @@ def __load_nc_file(self): nc = load_from_https(self.uri) self.filename = self.uri self.ds = nc[ncvar] + print("Loaded dataset", self.ds) def __get_missing_attributes(self): if self.ds is None: @@ -366,21 +367,21 @@ def method(self, value): def mean(self, axis=None): self._method = "mean" if axis is not None: - self.axis = axis + self._axis = axis return self def min(self, axis=None): self._method = "min" if axis is not None: - self.axis = axis + self._axis = axis return self def max(self, axis=None): self._method = "max" if axis is not None: - self.axis = axis + self._axis = axis return self @property From 7298051e7fd75e41f2ea3e208639b1989db77749 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Dec 2025 14:40:54 +0000 Subject: [PATCH 10/22] turn on some screen printing --- activestorage/reductionist.py | 1 + 1 file changed, 1 insertion(+) diff --git a/activestorage/reductionist.py b/activestorage/reductionist.py index 4cc4089d..e7cff1c0 100644 --- a/activestorage/reductionist.py +++ b/activestorage/reductionist.py @@ -88,6 +88,7 @@ def reduce_chunk(session, chunk_selection, axis, storage_type=storage_type) + print(f"Reductionist request data dictionary: {request_data}") if DEBUG: print(f"Reductionist request data dictionary: {request_data}") api_operation = "sum" if operation == "mean" else operation or "select" From 5ef6146dfca3c3aceab9d8b26b54cb03a7c094e3 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Dec 2025 14:41:13 +0000 Subject: [PATCH 11/22] ix test --- tests/unit/test_active_axis.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/unit/test_active_axis.py b/tests/unit/test_active_axis.py index 05712394..d3d62cf1 100644 --- a/tests/unit/test_active_axis.py +++ b/tests/unit/test_active_axis.py @@ -89,18 +89,19 @@ def test_active_axis_format_new_api(): active1 = Active(rfile, ncvar) active2 = Active(rfile, ncvar) - x1 = active2.mean(axis=[0, 2])[...] - assert active2.axis == [0, 2] + x1 = active2.mean(axis=(0, 2))[...] + assert active2._axis == (0, 2) x2 = active2.mean(axis=(-1, -3))[...] - assert active2.axis == (-1, -3) + assert active2._axis == (-1, -3) assert x1.shape == x2.shape assert (x1.mask == x2.mask).all() assert np.ma.allclose(x1, x2) - xmin = active2.min(axis=[0, 2])[...] - xmax = active2.min(axis=[0, 2])[...] - assert xmin == xmax == [[[198.82859802246094]]] + xmin = active2.min(axis=(0, 2))[...] + xmax = active2.max(axis=(0, 2))[...] + assert xmin[0][0][0] == 209.44680786132812 + assert xmax[0][0][0] == 255.54661560058594 def test_active_axis_format_2(): From f7f69499c66c16d48120d91ba5a4298e961816eb Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Dec 2025 14:41:32 +0000 Subject: [PATCH 12/22] final version of not working Reductionist test --- tests/test_real_s3_with_axes.py | 36 +++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/tests/test_real_s3_with_axes.py b/tests/test_real_s3_with_axes.py index b81c221f..364f83c1 100644 --- a/tests/test_real_s3_with_axes.py +++ b/tests/test_real_s3_with_axes.py @@ -30,25 +30,57 @@ def build_active(): return active +## Active loads a 4dim dataset +## Loaded dataset +## default axis arg (when axis=None): 'axis': (0, 1, 2, 3) + def test_no_axis(): + """ + Fails: it should pass: 'axis': (0, 1, 2, 3) default + are fine! + + activestorage.reductionist.ReductionistError: Reductionist error: HTTP 400: {"error": {"message": "request data is not valid", "caused_by": ["__all__: Validation error: Number of reduction axes must be less than length of shape - to reduce over all axes omit the axis field completely [{}]"]}} + """ active = build_active() result = active.min()[:] assert result == [[[[164.8125]]]] def test_no_axis_2(): + """ + Fails: it should pass: 'axis': (0, 1, 2, 3) default + are fine! + + activestorage.reductionist.ReductionistError: Reductionist error: HTTP 400: {"error": {"message": "request data is not valid", "caused_by": ["__all__: Validation error: Number of reduction axes must be less than length of shape - to reduce over all axes omit the axis field completely [{}]"]}} + """ active = build_active() result = active.min(axis=())[:] assert result == [[[[164.8125]]]] +def test_axis_0(): + """Fails: activestorage.reductionist.ReductionistError: Reductionist error: HTTP 502: -""" + active = build_active() + result = active.min(axis=(0, ))[:] + assert result == [[[[164.8125]]]] + + def test_axis_0_1(): + """Fails: activestorage.reductionist.ReductionistError: Reductionist error: HTTP 502: -""" active = build_active() result = active.min(axis=(0, 1))[:] assert result == [[[[164.8125]]]] -def test_no_axis_1(): +def test_axis_1(): + """Fails: activestorage.reductionist.ReductionistError: Reductionist error: HTTP 502: -""" active = build_active() - result = active.min(axis=(1))[:] + result = active.min(axis=(1, ))[:] assert result == [[[[164.8125]]]] + + +def test_axis_0_1_2(): + """Passes fine.""" + active = build_active() + result = active.min(axis=(0, 1, 2))[:] + assert result[0][0][0][0] == 171.05126953125 From 72f12ac1f22a40e30167094dfcd97e7f5911a3a1 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Dec 2025 15:01:41 +0000 Subject: [PATCH 13/22] too many blank lines --- activestorage/active.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/activestorage/active.py b/activestorage/active.py index 220212b7..6d1bfe59 100644 --- a/activestorage/active.py +++ b/activestorage/active.py @@ -363,21 +363,18 @@ def method(self, value): self._method = value - def mean(self, axis=None): self._method = "mean" if axis is not None: self._axis = axis return self - def min(self, axis=None): self._method = "min" if axis is not None: self._axis = axis return self - def max(self, axis=None): self._method = "max" if axis is not None: From d87005af52100b270b935964519e599f6b085cc7 Mon Sep 17 00:00:00 2001 From: Max Norton Date: Wed, 21 Jan 2026 09:51:16 +0000 Subject: [PATCH 14/22] Change the Reductionist API to return a JSON payload, ths removes all the headers such as x-activestorage-count in favour of returning their data as part of the JSON payload. --- activestorage/reductionist.py | 9 +++++---- tests/unit/test_reductionist.py | 15 +++++++++------ 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/activestorage/reductionist.py b/activestorage/reductionist.py index e7cff1c0..17109a50 100644 --- a/activestorage/reductionist.py +++ b/activestorage/reductionist.py @@ -235,15 +235,16 @@ def request(session: requests.Session, url: str, request_data: dict): def decode_result(response): """Decode a successful response, return as a 2-tuple of (numpy array or scalar, count).""" - dtype = response.headers['x-activestorage-dtype'] - shape = json.loads(response.headers['x-activestorage-shape']) + reduction_result = json.loads(response.content) + dtype = reduction_result['dtype'] + shape = reduction_result['shape'] # Result - result = np.frombuffer(response.content, dtype=dtype) + result = np.frombuffer(bytes(reduction_result['bytes']), dtype=dtype) result = result.reshape(shape) # Counts - count = json.loads(response.headers['x-activestorage-count']) + count = reduction_result['count'] # TODO: When reductionist is ready, we need to fix 'count' # Mask the result diff --git a/tests/unit/test_reductionist.py b/tests/unit/test_reductionist.py index 3b06670e..27b386c9 100644 --- a/tests/unit/test_reductionist.py +++ b/tests/unit/test_reductionist.py @@ -11,15 +11,18 @@ def make_response(content, status_code, dtype=None, shape=None, count=None): - response = requests.Response() - response._content = content - response.status_code = status_code + reduction_result = { + "bytes": content + } if dtype: - response.headers["x-activestorage-dtype"] = dtype + reduction_result["dtype"] = dtype if shape: - response.headers["x-activestorage-shape"] = shape + reduction_result["shape"] = shape if count: - response.headers["x-activestorage-count"] = count + reduction_result["count"] = count + response = requests.Response() + response._content = json.dumps(reduction_result) + response.status_code = status_code return response From b2e6e498abb090f5c7e700f7bf2c11f78c41b12a Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 21 Jan 2026 14:02:43 +0000 Subject: [PATCH 15/22] add json import --- tests/unit/test_reductionist.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/test_reductionist.py b/tests/unit/test_reductionist.py index 27b386c9..ec54228b 100644 --- a/tests/unit/test_reductionist.py +++ b/tests/unit/test_reductionist.py @@ -2,6 +2,7 @@ import sys from unittest import mock +import json import numcodecs import numpy as np import pytest From 6857d22eef0d995386b0e548075afa0e4625b760 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 21 Jan 2026 14:07:36 +0000 Subject: [PATCH 16/22] add some prints for response and sizeof --- activestorage/reductionist.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/activestorage/reductionist.py b/activestorage/reductionist.py index 17109a50..23c582b6 100644 --- a/activestorage/reductionist.py +++ b/activestorage/reductionist.py @@ -236,6 +236,8 @@ def request(session: requests.Session, url: str, request_data: dict): def decode_result(response): """Decode a successful response, return as a 2-tuple of (numpy array or scalar, count).""" reduction_result = json.loads(response.content) + print("Reduction result: ", reduction_result) + print("Reduction result size: ", sys.getsizeof(reduction_result)) dtype = reduction_result['dtype'] shape = reduction_result['shape'] From 432fc4f8a97c58e0c2097dcb308174f5cee4c5f8 Mon Sep 17 00:00:00 2001 From: Max Norton Date: Wed, 21 Jan 2026 16:43:34 +0000 Subject: [PATCH 17/22] Use response.json() as Reductionist should return application/json --- activestorage/reductionist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activestorage/reductionist.py b/activestorage/reductionist.py index 23c582b6..4b083c8f 100644 --- a/activestorage/reductionist.py +++ b/activestorage/reductionist.py @@ -235,7 +235,7 @@ def request(session: requests.Session, url: str, request_data: dict): def decode_result(response): """Decode a successful response, return as a 2-tuple of (numpy array or scalar, count).""" - reduction_result = json.loads(response.content) + reduction_result = response.json() print("Reduction result: ", reduction_result) print("Reduction result size: ", sys.getsizeof(reduction_result)) dtype = reduction_result['dtype'] From e371ceee5936f479631955a0ae5d7e79c309a6e5 Mon Sep 17 00:00:00 2001 From: Max Norton Date: Thu, 22 Jan 2026 09:58:46 +0000 Subject: [PATCH 18/22] Fix Reductionist unit tests --- activestorage/reductionist.py | 4 ++-- tests/unit/test_reductionist.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/activestorage/reductionist.py b/activestorage/reductionist.py index 4b083c8f..3c6809e8 100644 --- a/activestorage/reductionist.py +++ b/activestorage/reductionist.py @@ -235,11 +235,11 @@ def request(session: requests.Session, url: str, request_data: dict): def decode_result(response): """Decode a successful response, return as a 2-tuple of (numpy array or scalar, count).""" - reduction_result = response.json() + reduction_result = json.loads(response.content) print("Reduction result: ", reduction_result) print("Reduction result size: ", sys.getsizeof(reduction_result)) dtype = reduction_result['dtype'] - shape = reduction_result['shape'] + shape = reduction_result['shape'] if "shape" in reduction_result else None # Result result = np.frombuffer(bytes(reduction_result['bytes']), dtype=dtype) diff --git a/tests/unit/test_reductionist.py b/tests/unit/test_reductionist.py index ec54228b..b1aa9831 100644 --- a/tests/unit/test_reductionist.py +++ b/tests/unit/test_reductionist.py @@ -13,7 +13,7 @@ def make_response(content, status_code, dtype=None, shape=None, count=None): reduction_result = { - "bytes": content + "bytes": list(content) } if dtype: reduction_result["dtype"] = dtype @@ -31,7 +31,7 @@ def make_response(content, status_code, dtype=None, shape=None, count=None): def test_reduce_chunk_defaults(mock_request): """Unit test for reduce_chunk with default arguments.""" result = np.int32(134351386) - response = make_response(result.tobytes(), 200, "int32", "[]", "2") + response = make_response(result.tobytes(), 200, "int32", [], 2) mock_request.return_value = response active_url = "https://s3.example.com" @@ -90,7 +90,7 @@ def test_reduce_chunk_defaults(mock_request): def test_reduce_chunk_compression(mock_request, compression, filters): """Unit test for reduce_chunk with compression and filter arguments.""" result = np.int32(134351386) - response = make_response(result.tobytes(), 200, "int32", "[]", "2") + response = make_response(result.tobytes(), 200, "int32", [], 2) mock_request.return_value = response active_url = "https://s3.example.com" @@ -206,7 +206,7 @@ def test_reduce_chunk_missing(mock_request, missing): reduce_arg, api_arg = missing result = np.float32(-42.) - response = make_response(result.tobytes(), 200, "float32", "[]", "2") + response = make_response(result.tobytes(), 200, "float32", [], 2) mock_request.return_value = response active_url = "https://s3.example.com" From 88fa26f708dfb269fdf09f4fc13353084d9fc4d1 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 23 Jan 2026 15:58:51 +0000 Subject: [PATCH 19/22] add small file test --- tests/test_real_s3_with_axes.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/test_real_s3_with_axes.py b/tests/test_real_s3_with_axes.py index 364f83c1..685e972a 100644 --- a/tests/test_real_s3_with_axes.py +++ b/tests/test_real_s3_with_axes.py @@ -6,6 +6,37 @@ S3_BUCKET = "bnl" +def build_active_small_file(): + """Run an integration test with real data off S3 but with a small file.""" + storage_options = { + 'key': "f2d55c6dcfc7618b2c34e00b58df3cef", + 'secret': "$/'#M{0{/4rVhp%n^(XeX$q@y#&(NM3W1->~N.Q6VP.5[@bLpi='nt]AfH)>78pT", + 'client_kwargs': {'endpoint_url': "https://uor-aces-o.s3-ext.jc.rl.ac.uk"}, # final proxy + } + active_storage_url = "https://reductionist.jasmin.ac.uk/" # Wacasoft new Reductionist + bigger_file = "CMIP6-test.nc" # tas; 15 (time) x 143 x 144 + + test_file_uri = os.path.join( + S3_BUCKET, + bigger_file + ) + print("S3 Test file path:", test_file_uri) + active = Active(test_file_uri, 'tas', storage_type="s3", + storage_options=storage_options, + active_storage_url=active_storage_url) + + active._version = 2 + + return active + + +def test_small_file_axis_0_1(): + """Fails: activestorage.reductionist.ReductionistError: Reductionist error: HTTP 502: -""" + active = build_active_small_file() + result = active.min(axis=(0, 1))[:] + assert result == [[[[164.8125]]]] + + def build_active(): """Run an integration test with real data off S3.""" storage_options = { From 4b4f3ad918352956bebc223580615967202f9227 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 23 Jan 2026 16:00:38 +0000 Subject: [PATCH 20/22] toss a print statement --- tests/test_real_s3_with_axes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_real_s3_with_axes.py b/tests/test_real_s3_with_axes.py index 685e972a..d463230a 100644 --- a/tests/test_real_s3_with_axes.py +++ b/tests/test_real_s3_with_axes.py @@ -34,6 +34,7 @@ def test_small_file_axis_0_1(): """Fails: activestorage.reductionist.ReductionistError: Reductionist error: HTTP 502: -""" active = build_active_small_file() result = active.min(axis=(0, 1))[:] + print("Reductionist final result", result) assert result == [[[[164.8125]]]] From dc41f7b5dc48cb9b95f9dd557dd2e9775e6101fa Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 23 Jan 2026 16:33:09 +0000 Subject: [PATCH 21/22] add validation vs numpy --- tests/test_real_s3_with_axes.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/test_real_s3_with_axes.py b/tests/test_real_s3_with_axes.py index d463230a..6eb2fb57 100644 --- a/tests/test_real_s3_with_axes.py +++ b/tests/test_real_s3_with_axes.py @@ -1,5 +1,6 @@ import os import numpy as np +import pyfive from activestorage.active import Active @@ -35,7 +36,22 @@ def test_small_file_axis_0_1(): active = build_active_small_file() result = active.min(axis=(0, 1))[:] print("Reductionist final result", result) - assert result == [[[[164.8125]]]] + assert min(result[0][0]) == 197.69595 + + +def test_small_file_axis_0_1_compare_with_numpy(): + """Fails: activestorage.reductionist.ReductionistError: Reductionist error: HTTP 502: -""" + active = build_active_small_file() + result = active.min(axis=(0, 1))[:] + print("Reductionist final result", result) + + # use numpy and local test data + ds = pyfive.File("tests/test_data/CMIP6-test.nc")["tas"] + minarr= np.min(ds[:], axis=(0, 1)) + print(len(minarr)) # 144 + print(min(minarr)) # 197.69595 + assert len(result[0][0]) == len(minarr) + assert min(result[0][0]) == min(minarr) def build_active(): From ae71547efbd6c51ac1054f686ead7e5007f86933 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 23 Jan 2026 17:26:23 +0000 Subject: [PATCH 22/22] betterify test --- tests/test_real_s3_with_axes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_real_s3_with_axes.py b/tests/test_real_s3_with_axes.py index 6eb2fb57..826103b7 100644 --- a/tests/test_real_s3_with_axes.py +++ b/tests/test_real_s3_with_axes.py @@ -47,11 +47,11 @@ def test_small_file_axis_0_1_compare_with_numpy(): # use numpy and local test data ds = pyfive.File("tests/test_data/CMIP6-test.nc")["tas"] - minarr= np.min(ds[:], axis=(0, 1)) + minarr= np.min(ds[:], axis=(0, 1), keepdims=True) print(len(minarr)) # 144 print(min(minarr)) # 197.69595 - assert len(result[0][0]) == len(minarr) - assert min(result[0][0]) == min(minarr) + assert np.min(result) == np.min(minarr) + np.testing.assert_array_equal(result, minarr) def build_active():