From 4d2bc2bb0ad52891376dc72820424d17eca80732 Mon Sep 17 00:00:00 2001 From: Slmtdhy <750641107@qq.com> Date: Tue, 5 Nov 2024 16:23:20 +0800 Subject: [PATCH 01/34] Added longtermdataset --- hydrodatasource/reader/data_source.py | 232 ++++++++++++++++++++++++++ 1 file changed, 232 insertions(+) diff --git a/hydrodatasource/reader/data_source.py b/hydrodatasource/reader/data_source.py index 7417621..69035ca 100644 --- a/hydrodatasource/reader/data_source.py +++ b/hydrodatasource/reader/data_source.py @@ -608,3 +608,235 @@ def read_area(self, gage_id_lst=None): def read_mean_prcp(self, gage_id_lst=None): """read mean precipitation of each basin/unit""" return self.read_attr_xrdataset(gage_id_lst, ["pre_mm_syr"]) + + +class LongTermDataset(SelfMadeHydroDataset): + def __init__(self, data_path, download=False, time_unit=None): + super(LongTermDataset,self).__init__(data_path) + if time_unit is None: + time_unit = ["1MS"] + if any(unit not in ["1h", "3h", "1D", "8D", "1MS"] for unit in time_unit): + raise ValueError( + "time_unit must be one of ['1h', '3h', '1D', '8D', '1MS']. We only support these time units now." + ) + # TODO: maybe starting with "s3://" is a better idea? + self.head = "minio" if "s3://" in data_path else "local" + super().__init__(data_path) + self.data_source_description = self.set_data_source_describe() + if download: + self.download_data_source() + self.camels_sites = self.read_site_info() + self.time_unit = time_unit + def read_site_info(self): + camels_file = self.data_source_description["ATTR_FILE"] + attrs = access_fs.spec_path(camels_file, head=self.head) + return attrs[["basin_id"]] + + def set_data_source_describe(self): + data_root_dir = self.data_source_dir + ts_dir = os.path.join(data_root_dir, "timeseries") + # we assume that each subdirectory in ts_dir represents a time unit + # In this subdirectory, there are csv files for each basin + if "s3://" in data_root_dir: + time_units_dir = [ + os.path.join(ts_dir, name) + for name in minio_file_list(ts_dir) + if is_minio_folder(os.path.join(ts_dir, name)) + ] + else: + time_units_dir = [ + os.path.join(ts_dir, name) + for name in os.listdir(ts_dir) + if os.path.isdir(os.path.join(ts_dir, name)) + ] + unit_files = [folder + "_units_info.json" for folder in time_units_dir] + attr_dir = os.path.join(data_root_dir, "attributes") + attr_file = os.path.join(attr_dir, "grdc_attributes1.csv") + shape_dir = os.path.join(data_root_dir, "shapes") + global_dir = os.path.join(data_root_dir, "attributes") + return collections.OrderedDict( + DATA_DIR=data_root_dir, + TS_DIRS=time_units_dir, + ATTR_DIR=attr_dir, + ATTR_FILE=attr_file, + UNIT_FILES=unit_files, + SHAPE_DIR=shape_dir, + GLOBAL_DIR=global_dir, + ) + def read_attr_xrdataset(self,gage_id_lst=None,var_lst=None, **kwargs): + region = kwargs.get("region", None) + + prefix_ = "" if region is None else region + "_" + if var_lst is None or len(var_lst) == 0: + return None + try: + attr = xr.open_dataset(os.path.join(CACHE_DIR, f"{prefix_}attributes.nc")) + except FileNotFoundError: + self.cache_xrdataset(time_units=self.time_unit) + attr = xr.open_dataset(os.path.join(CACHE_DIR, f"{prefix_}attributes.nc")) + # return attr.sel(basin=gage_id_lst) + # return attr + gage_id_lst=[str(i) for i in gage_id_lst] + return attr[var_lst].sel(basin=gage_id_lst) + def read_global_data(self, object_ids: list = None, t_range_list: list = None) -> dict: + """ + 读取全球数据，返回一个字典，其中每个键是时间单位，每个值是一个 xarray.Dataset。 + """ + # self.cache_global_dataset(object_ids, t_range_list) + + + if not os.path.exists(os.path.join(CACHE_DIR, "global_data.nc")): + self.cache_global_dataset(object_ids, t_range_list) + netcdf_file = os.path.join(CACHE_DIR, "global_data.nc") + else: + netcdf_file = os.path.join(CACHE_DIR, "global_data.nc") + # print(netcdf_file) + ds = xr.open_dataset(netcdf_file) + + result_dict = {} + result_dict = {"global_dataset": ds} + + return result_dict + + def cache_global_dataset(self, object_ids: list = None, t_range_list: list = None): + """ + 读取 CSV 数据并将其转换为 xarray 数据集并保存为 NetCDF 文件。 + """ + global_data_file = os.path.join(self.data_source_description["GLOBAL_DIR"], "global.csv") + if not os.path.exists(global_data_file): + raise FileNotFoundError(f"Global data file not found at {global_data_file}") + global_data = pd.read_csv(global_data_file, parse_dates=["time"]) + start_time = t_range_list[0] + end_time = t_range_list[-1] + mask = (global_data["time"] >= start_time) & (global_data["time"] <= end_time) + filtered_data = global_data[mask] + + time_col = pd.to_datetime(filtered_data['time']) + + variable_names = filtered_data.columns.drop('time').tolist() + ds = xr.Dataset( + coords={ + "basin": object_ids, + "time": time_col + } + ) + + for i, var in enumerate(variable_names): + # 由于没有单位信息，我们只传递数据 + data_array = np.tile(filtered_data[var].values, (len(object_ids), 1)) # 复制变量数据以适应 (basin, time) + + # 添加变量到 Dataset + ds[var] = (("basin", "time"), data_array) + + # 保存为 NetCDF 文件 + ds.to_netcdf(os.path.join(CACHE_DIR, "global_data.nc")) + print("NetCDF 文件已成功保存为 global_data.nc") + del global_data + + def cache_timeseries_xrdataset(self, region=None, t_range=None, **kwargs): + """Save all timeseries data in separate NetCDF files for each time unit. + + Parameters + ---------- + region : str, optional + A prefix used in cache file, by default None + t_range : list, optional + Time range for the data, by default ["1980-01-01", "2023-12-31"] + kwargs : dict, optional + batchsize -- Number of basins to process per batch, by default 100 + time_units -- List of time units to process, by default None + start0101_freq -- for freq setting, if the start date is 01-01, set True, by default False + """ + batchsize = kwargs.get("batchsize", 100) + time_units = kwargs.get("time_units", self.time_unit) or [ + "1D" + ] # Default to ["1D"] if not specified or if time_units is None + start0101_freq = kwargs.get("start0101_freq", False) + + variables = self.get_timeseries_cols() + basins = self.camels_sites["basin_id"] + # Define the generator function for batching + def data_generator(basins, batch_size): + for i in range(0, len(basins), batch_size): + yield basins[i : i + batch_size] + for time_unit in time_units: + if t_range is None: + if time_unit != "3h": + t_range = ["1951-01-01", "2010-12-31"] + else: + t_range = ["1951-01-01", "2010-12-31"] + + # Generate the time range specific to the time unit + if start0101_freq: + times = ( + generate_start0101_time_range( + start_time=t_range[0], end_time=t_range[-1], freq=time_unit + ) + .strftime("%Y-%m-%d %H:%M:%S") + .tolist() + ) + else: + times = ( + pd.date_range(start=t_range[0], end=t_range[-1], freq=time_unit) + .strftime("%Y-%m-%d %H:%M:%S") + .tolist() + ) + # Retrieve the correct units information for this time unit + unit_file = next( + file + for file in self.data_source_description["UNIT_FILES"] + if time_unit in file + ) + if "s3://" in unit_file: + with conf.FS.open(unit_file, mode="rb") as fp: + units_info = json.load(fp) + else: + units_info = hydro_file.unserialize_json(unit_file) + + for basin_batch in data_generator(basins, batchsize): + basin_batch = [int(basin) for basin in basin_batch if not pd.isna(basin)] + data = self.read_timeseries( + object_ids=basin_batch, + t_range_list=t_range, + relevant_cols=variables[ + time_unit + ], # Ensure we use the right columns for the time unit + time_units=[ + time_unit + ], # Pass the time unit to ensure correct data retrieval + start0101_freq=start0101_freq, + ) + + dataset = xr.Dataset( + data_vars={ + variables[time_unit][i]: ( + ["basin", "time"], + data[time_unit][:, :, i], + {"units": units_info[variables[time_unit][i]]}, + ) + for i in range(len(variables[time_unit])) + }, + coords={ + "basin": basin_batch, + "time": pd.to_datetime(times), + }, + ) + + # Save the dataset to a NetCDF file for the current batch and time unit + prefix_ = "" if region is None else region + "_" + batch_file_path = os.path.join( + CACHE_DIR, + f"{prefix_}timeseries_{time_unit}_batch_{basin_batch[0]}_{basin_batch[-1]}.nc", + ) + dataset.to_netcdf(batch_file_path) + + # Release memory by deleting the dataset + del dataset + del data + + def cache_xrdataset(self,region=None, t_range=None, time_units=None): + """Save all data in a netcdf file in the cache directory""" + self.cache_attributes_xrdataset(region=region) + self.cache_timeseries_xrdataset( + region=region, t_range=t_range, time_units=time_units + ) \ No newline at end of file From a53bbbffae580affebcb1220e41481095dc3b9ef Mon Sep 17 00:00:00 2001 From: Slmtdhy <750641107@qq.com> Date: Tue, 12 Nov 2024 14:08:26 +0800 Subject: [PATCH 02/34] Change the default region to "Caravan" --- hydrodatasource/reader/data_source.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hydrodatasource/reader/data_source.py b/hydrodatasource/reader/data_source.py index 69035ca..3f557f4 100644 --- a/hydrodatasource/reader/data_source.py +++ b/hydrodatasource/reader/data_source.py @@ -161,7 +161,7 @@ def read_timeseries( A dictionary containing data with different time scales. """ time_units = kwargs.get("time_units", ["1D"]) - region = kwargs.get("region", None) + region = kwargs.get("region", "Caravan") start0101_freq = kwargs.get("start0101_freq", False) results = {} @@ -204,7 +204,7 @@ def read_timeseries( prefix_ = "" if region is None else region + "_" ts_file = os.path.join( ts_dir, - prefix_ + object_ids[k] + ".csv", + prefix_ + str(object_ids[k]) + ".csv", ) if "s3://" in ts_file: with conf.FS.open(ts_file, mode="rb") as f: From 72f5ec5494a0861a647df20fb54e270f2e6a02bd Mon Sep 17 00:00:00 2001 From: Slmtdhy <750641107@qq.com> Date: Wed, 13 Nov 2024 18:09:53 +0800 Subject: [PATCH 03/34] Change the unit of average precipitation to "mm/M" and add unit conversion logic; optimize the selection logic of long-term data sets --- hydrodatasource/reader/data_source.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/hydrodatasource/reader/data_source.py b/hydrodatasource/reader/data_source.py index bffbb7e..85685ab 100644 --- a/hydrodatasource/reader/data_source.py +++ b/hydrodatasource/reader/data_source.py @@ -605,7 +605,7 @@ def read_area(self, gage_id_lst=None): """read area of each basin/unit""" return self.read_attr_xrdataset(gage_id_lst, ["area"]) - def read_mean_prcp(self, gage_id_lst=None, unit="mm/d"): + def read_mean_prcp(self, gage_id_lst=None, unit="mm/M"): """read mean precipitation of each basin default unit is mm/d, but one can chose other units and we will convert the unit to the specified unit @@ -632,6 +632,8 @@ def read_mean_prcp(self, gage_id_lst=None, unit="mm/d"): converted_data = da / (8760 / 3) elif unit in ["mm/8d", "mm/8day"]: converted_data = da / (365 / 8) + elif unit in ["mm/M"]: + converted_data = da / 12 else: raise ValueError( "unit must be one of ['mm/d', 'mm/day', 'mm/h', 'mm/hour', 'mm/3h', 'mm/3hour', 'mm/8d', 'mm/8day']" @@ -710,23 +712,17 @@ def read_attr_xrdataset(self, gage_id_lst=None, var_lst=None, **kwargs): except FileNotFoundError: self.cache_xrdataset(time_units=self.time_unit) attr = xr.open_dataset(os.path.join(CACHE_DIR, f"{prefix_}attributes.nc")) - # return attr.sel(basin=gage_id_lst) - # return attr + attr = attr.sel(basin=~attr.indexes['basin'].duplicated()) gage_id_lst = [str(i) for i in gage_id_lst] return attr[var_lst].sel(basin=gage_id_lst) def read_global_data( self, object_ids: list = None, t_range_list: list = None ) -> dict: - """ - 读取全球数据，返回一个字典，其中每个键是时间单位，每个值是一个 xarray.Dataset。 - """ - # self.cache_global_dataset(object_ids, t_range_list) if not os.path.exists(os.path.join(CACHE_DIR, "global_data.nc")): self.cache_global_dataset(object_ids, t_range_list) netcdf_file = os.path.join(CACHE_DIR, "global_data.nc") - # print(netcdf_file) ds = xr.open_dataset(netcdf_file) return {"global_dataset": ds} @@ -765,6 +761,7 @@ def cache_global_dataset(self, object_ids: list = None, t_range_list: list = Non print("NetCDF 文件已成功保存为 global_data.nc") del global_data + def cache_timeseries_xrdataset(self, region=None, t_range=None, **kwargs): """Save all timeseries data in separate NetCDF files for each time unit. From 2d45dd08ad9a0f37f0ec1dbe4f73d409df09cd64 Mon Sep 17 00:00:00 2001 From: Slmtdhy <750641107@qq.com> Date: Thu, 14 Nov 2024 16:26:49 +0800 Subject: [PATCH 04/34] Fixed bug reading basicid --- hydrodatasource/reader/data_source.py | 117 ++------------------------ 1 file changed, 6 insertions(+), 111 deletions(-) diff --git a/hydrodatasource/reader/data_source.py b/hydrodatasource/reader/data_source.py index 85685ab..e2780ac 100644 --- a/hydrodatasource/reader/data_source.py +++ b/hydrodatasource/reader/data_source.py @@ -161,7 +161,7 @@ def read_timeseries( A dictionary containing data with different time scales. """ time_units = kwargs.get("time_units", ["1D"]) - region = kwargs.get("region", "Caravan") + region = kwargs.get("region", None) start0101_freq = kwargs.get("start0101_freq", False) results = {} @@ -429,9 +429,9 @@ def data_generator(basins, batch_size): for time_unit in time_units: if t_range is None: if time_unit != "3h": - t_range = ["1980-01-01", "2023-12-31"] + t_range = ["1950-01-01", "2023-12-31"] else: - t_range = ["1980-01-01 01", "2023-12-31 22"] + t_range = ["1950-01-01 01", "2023-12-31 22"] # Generate the time range specific to the time unit if start0101_freq: @@ -762,114 +762,9 @@ def cache_global_dataset(self, object_ids: list = None, t_range_list: list = Non del global_data - def cache_timeseries_xrdataset(self, region=None, t_range=None, **kwargs): - """Save all timeseries data in separate NetCDF files for each time unit. - - Parameters - ---------- - region : str, optional - A prefix used in cache file, by default None - t_range : list, optional - Time range for the data, by default ["1980-01-01", "2023-12-31"] - kwargs : dict, optional - batchsize -- Number of basins to process per batch, by default 100 - time_units -- List of time units to process, by default None - start0101_freq -- for freq setting, if the start date is 01-01, set True, by default False - """ - batchsize = kwargs.get("batchsize", 100) - time_units = kwargs.get("time_units", self.time_unit) or [ - "1D" - ] # Default to ["1D"] if not specified or if time_units is None - start0101_freq = kwargs.get("start0101_freq", False) - - variables = self.get_timeseries_cols() - basins = self.camels_sites["basin_id"] - - # Define the generator function for batching - def data_generator(basins, batch_size): - for i in range(0, len(basins), batch_size): - yield basins[i : i + batch_size] - - for time_unit in time_units: - if t_range is None: - if time_unit != "3h": - t_range = ["1951-01-01", "2010-12-31"] - else: - t_range = ["1951-01-01", "2010-12-31"] - - # Generate the time range specific to the time unit - if start0101_freq: - times = ( - generate_start0101_time_range( - start_time=t_range[0], end_time=t_range[-1], freq=time_unit - ) - .strftime("%Y-%m-%d %H:%M:%S") - .tolist() - ) - else: - times = ( - pd.date_range(start=t_range[0], end=t_range[-1], freq=time_unit) - .strftime("%Y-%m-%d %H:%M:%S") - .tolist() - ) - # Retrieve the correct units information for this time unit - unit_file = next( - file - for file in self.data_source_description["UNIT_FILES"] - if time_unit in file - ) - if "s3://" in unit_file: - with conf.FS.open(unit_file, mode="rb") as fp: - units_info = json.load(fp) - else: - units_info = hydro_file.unserialize_json(unit_file) - - for basin_batch in data_generator(basins, batchsize): - basin_batch = [ - int(basin) for basin in basin_batch if not pd.isna(basin) - ] - data = self.read_timeseries( - object_ids=basin_batch, - t_range_list=t_range, - relevant_cols=variables[ - time_unit - ], # Ensure we use the right columns for the time unit - time_units=[ - time_unit - ], # Pass the time unit to ensure correct data retrieval - start0101_freq=start0101_freq, - ) - - dataset = xr.Dataset( - data_vars={ - variables[time_unit][i]: ( - ["basin", "time"], - data[time_unit][:, :, i], - {"units": units_info[variables[time_unit][i]]}, - ) - for i in range(len(variables[time_unit])) - }, - coords={ - "basin": basin_batch, - "time": pd.to_datetime(times), - }, - ) - - # Save the dataset to a NetCDF file for the current batch and time unit - prefix_ = "" if region is None else region + "_" - batch_file_path = os.path.join( - CACHE_DIR, - f"{prefix_}timeseries_{time_unit}_batch_{basin_batch[0]}_{basin_batch[-1]}.nc", - ) - dataset.to_netcdf(batch_file_path) - - # Release memory by deleting the dataset - del dataset - del data - def cache_xrdataset(self, region=None, t_range=None, time_units=None): """Save all data in a netcdf file in the cache directory""" self.cache_attributes_xrdataset(region=region) - self.cache_timeseries_xrdataset( - region=region, t_range=t_range, time_units=time_units - ) + # self.cache_timeseries_xrdataset( + # region=region, t_range=t_range, time_units=time_units + # ) From f8d8f1fb2a621f15328d666c23a3efe856b0486c Mon Sep 17 00:00:00 2001 From: Slmtdhy <750641107@qq.com> Date: Mon, 30 Dec 2024 09:09:04 +0800 Subject: [PATCH 05/34] refine datasource --- hydrodatasource/reader/data_source.py | 74 ++++++++++----------------- 1 file changed, 27 insertions(+), 47 deletions(-) diff --git a/hydrodatasource/reader/data_source.py b/hydrodatasource/reader/data_source.py index e2780ac..2a4f493 100644 --- a/hydrodatasource/reader/data_source.py +++ b/hydrodatasource/reader/data_source.py @@ -71,10 +71,10 @@ def __init__(self, data_path, download=False, time_unit=None): _description_, by default """ if time_unit is None: - time_unit = ["1D"] - if any(unit not in ["1h", "3h", "1D", "8D"] for unit in time_unit): + time_unit = ["1MS"] + if any(unit not in ["1h", "3h", "1D", "8D", "1MS"] for unit in time_unit): raise ValueError( - "time_unit must be one of ['1h', '3h', '1D', '8D']. We only support these time units now." + "time_unit must be one of ['1h', '3h', '1D', '8D','1MS']. We only support these time units now." ) # TODO: maybe starting with "s3://" is a better idea? self.head = "minio" if "s3://" in data_path else "local" @@ -87,7 +87,7 @@ def __init__(self, data_path, download=False, time_unit=None): @property def streamflow_unit(self): - unit_mapping = {"1h": "mm/h", "3h": "mm/3h", "1D": "mm/d"} + unit_mapping = {"1h": "mm/h", "3h": "mm/3h", "1D": "mm/d", "1MS": "mm/M"} return {unit: unit_mapping[unit] for unit in self.time_unit} def get_name(self): @@ -648,21 +648,7 @@ def read_mean_prcp(self, gage_id_lst=None, unit="mm/M"): class LongTermDataset(SelfMadeHydroDataset): def __init__(self, data_path, download=False, time_unit=None): - super(LongTermDataset, self).__init__(data_path) - if time_unit is None: - time_unit = ["1MS"] - if any(unit not in ["1h", "3h", "1D", "8D", "1MS"] for unit in time_unit): - raise ValueError( - "time_unit must be one of ['1h', '3h', '1D', '8D', '1MS']. We only support these time units now." - ) - # TODO: maybe starting with "s3://" is a better idea? - self.head = "minio" if "s3://" in data_path else "local" super().__init__(data_path) - self.data_source_description = self.set_data_source_describe() - if download: - self.download_data_source() - self.camels_sites = self.read_site_info() - self.time_unit = time_unit def read_site_info(self): camels_file = self.data_source_description["ATTR_FILE"] @@ -688,7 +674,7 @@ def set_data_source_describe(self): ] unit_files = [folder + "_units_info.json" for folder in time_units_dir] attr_dir = os.path.join(data_root_dir, "attributes") - attr_file = os.path.join(attr_dir, "grdc_attributes1.csv") + attr_file = os.path.join(attr_dir, "grdc_attr_178.csv") shape_dir = os.path.join(data_root_dir, "shapes") global_dir = os.path.join(data_root_dir, "attributes") return collections.OrderedDict( @@ -701,38 +687,40 @@ def set_data_source_describe(self): GLOBAL_DIR=global_dir, ) - def read_attr_xrdataset(self, gage_id_lst=None, var_lst=None, **kwargs): - region = kwargs.get("region", None) - - prefix_ = "" if region is None else region + "_" - if var_lst is None or len(var_lst) == 0: - return None - try: - attr = xr.open_dataset(os.path.join(CACHE_DIR, f"{prefix_}attributes.nc")) - except FileNotFoundError: - self.cache_xrdataset(time_units=self.time_unit) - attr = xr.open_dataset(os.path.join(CACHE_DIR, f"{prefix_}attributes.nc")) - attr = attr.sel(basin=~attr.indexes['basin'].duplicated()) - gage_id_lst = [str(i) for i in gage_id_lst] - return attr[var_lst].sel(basin=gage_id_lst) + # def read_attr_xrdataset(self, gage_id_lst=None, var_lst=None, **kwargs): + # region = kwargs.get("region", None) + + # prefix_ = "" if region is None else region + "_" + # if var_lst is None or len(var_lst) == 0: + # return None + # try: + # attr = xr.open_dataset(os.path.join(CACHE_DIR, f"{prefix_}attributes.nc")) + # except FileNotFoundError: + # self.cache_xrdataset(time_units=self.time_unit) + # attr = xr.open_dataset(os.path.join(CACHE_DIR, f"{prefix_}attributes.nc")) + # attr = attr.sel(basin=~attr.indexes['basin'].duplicated()) + # gage_id_lst = [str(i) for i in gage_id_lst] + # return attr[var_lst].sel(basin=gage_id_lst) def read_global_data( - self, object_ids: list = None, t_range_list: list = None + self, object_ids: list = None, t_range_list: list = None, var_lst: list = None ) -> dict: - + if var_lst is None or len(var_lst) == 0: + return None if not os.path.exists(os.path.join(CACHE_DIR, "global_data.nc")): self.cache_global_dataset(object_ids, t_range_list) netcdf_file = os.path.join(CACHE_DIR, "global_data.nc") - ds = xr.open_dataset(netcdf_file) + global_data = xr.open_dataset(netcdf_file) - return {"global_dataset": ds} + return global_data[var_lst].sel(basin=object_ids) + # return {"global_dataset": global_data} def cache_global_dataset(self, object_ids: list = None, t_range_list: list = None): """ 读取 CSV 数据并将其转换为 xarray 数据集并保存为 NetCDF 文件。 """ global_data_file = os.path.join( - self.data_source_description["GLOBAL_DIR"], "global.csv" + self.data_source_description["GLOBAL_DIR"], "global_data.csv" ) if not os.path.exists(global_data_file): raise FileNotFoundError(f"Global data file not found at {global_data_file}") @@ -748,20 +736,12 @@ def cache_global_dataset(self, object_ids: list = None, t_range_list: list = Non ds = xr.Dataset(coords={"basin": object_ids, "time": time_col}) for var in variable_names: - # 由于没有单位信息，我们只传递数据 - data_array = np.tile( - filtered_data[var].values, (len(object_ids), 1) - ) # 复制变量数据以适应 (basin, time) - - # 添加变量到 Dataset + data_array = np.tile(filtered_data[var].values, (len(object_ids), 1)) ds[var] = (("basin", "time"), data_array) - # 保存为 NetCDF 文件 ds.to_netcdf(os.path.join(CACHE_DIR, "global_data.nc")) - print("NetCDF 文件已成功保存为 global_data.nc") del global_data - def cache_xrdataset(self, region=None, t_range=None, time_units=None): """Save all data in a netcdf file in the cache directory""" self.cache_attributes_xrdataset(region=region) From 98f03eb85909aee3a93125320d0314a07f085d0b Mon Sep 17 00:00:00 2001 From: mahaoran2022 <965297063@qq.com> Date: Thu, 20 Nov 2025 17:34:26 +0800 Subject: [PATCH 06/34] tg_gnn test verson0.1 --- ...02\346\225\260\350\257\264\346\230\216.md" | 176 +++++++++ ...45\345\217\243\350\257\264\346\230\216.md" | 225 +++++++++++ example_tg_usage.py | 168 +++++++++ hydrodatasource/reader/data_source.py | 353 ++++++++++++++++++ ...02\346\225\260\350\257\264\346\230\216.md" | 195 ++++++++++ test_tg_hydro_datasource.py | 334 +++++++++++++++++ 6 files changed, 1451 insertions(+) create mode 100644 "TgHydroDatasource_\345\217\202\346\225\260\350\257\264\346\230\216.md" create mode 100644 "TgHydroDatasource_\350\276\223\345\207\272\346\216\245\345\217\243\350\257\264\346\230\216.md" create mode 100644 example_tg_usage.py create mode 100644 "read_lstm_predictions_\345\217\202\346\225\260\350\257\264\346\230\216.md" create mode 100644 test_tg_hydro_datasource.py diff --git "a/TgHydroDatasource_\345\217\202\346\225\260\350\257\264\346\230\216.md" "b/TgHydroDatasource_\345\217\202\346\225\260\350\257\264\346\230\216.md" new file mode 100644 index 0000000..8225755 --- /dev/null +++ "b/TgHydroDatasource_\345\217\202\346\225\260\350\257\264\346\230\216.md" @@ -0,0 +1,176 @@ +# TgHydroDatasource 参数说明文档 + +## 构造函数参数 + +### TgHydroDatasource 类的入参 + +```python +def __init__(self, data_path, download=False, time_unit=None, dataset_name=None, **kwargs): +``` + +### 参数详细说明 + +#### 1. 必需参数 + +| 参数名 | 类型 | 说明 | 示例 | +|--------|------|------|------| +| `data_path` | str | 数据根目录路径，可以是本地路径或S3路径 | `/path/to/data` 或 `s3://bucket/data` | + +#### 2. 可选参数 + +| 参数名 | 类型 | 默认值 | 说明 | 示例 | +|--------|------|--------|------|------| +| `download` | bool | False | 是否下载数据（如果数据源支持） | `True` 或 `False` | +| `time_unit` | list | ["1D"] | 时间单位列表，支持的单位有：["1h", "3h", "1D", "8D"] | `["1D"]` 或 `["1h", "3h"]` | +| `dataset_name` | str | None | 数据集名称，用于缓存文件命名等 | `"my_tg_dataset"` | +| `**kwargs` | dict | {} | 其他关键字参数，会传递给父类 | `{"version": "v1.0"}` | + +#### 3. kwargs 中支持的参数 + +| 参数名 | 类型 | 默认值 | 说明 | 示例 | +|--------|------|--------|------|------| +| `version` | str | None | 数据集版本号 | `"v2.0"` | + +## 参数传递机制 + +### 1. 继承关系 +``` +HydroData (基类) + ↓ +SelfMadeHydroDataset (父类) + ↓ +TgHydroDatasource (子类) +``` + +### 2. 参数传递流程 + +1. **TgHydroDatasource** 接收所有参数 +2. 通过 `super().__init__(data_path, download, time_unit, dataset_name, **kwargs)` 调用父类 +3. **SelfMadeHydroDataset** 处理自己的参数，再通过 `super().__init__(data_path)` 调用基类 +4. **HydroData** 基类只接收 `data_path` 参数 + +### 3. 参数处理详情 + +#### 在 SelfMadeHydroDataset 中： +```python +def __init__(self, data_path, download=False, time_unit=None, dataset_name=None, **kwargs): + # 参数验证和默认值设置 + if time_unit is None: + time_unit = ["1D"] + if any(unit not in ["1h", "3h", "1D", "8D"] for unit in time_unit): + raise ValueError("time_unit must be one of ['1h', '3h', '1D', '8D']") + + # 设置存储类型（本地或minio） + self.head = "minio" if "s3://" in data_path else "local" + + # 调用基类 + super().__init__(data_path) + + # 设置实例属性 + self.data_source_description = self.set_data_source_describe() + self.time_unit = time_unit + self.dataset_name = dataset_name + self.version = kwargs.get("version", None) # 从kwargs中提取version + + # 其他初始化 + if download: + self.download_data_source() + self.camels_sites = self.read_site_info() +``` + +#### 在 TgHydroDatasource 中： +```python +def __init__(self, data_path, download=False, time_unit=None, dataset_name=None, **kwargs): + # 调用父类初始化（所有参数都传递给父类） + super().__init__(data_path, download, time_unit, dataset_name, **kwargs) + + # TgHydroDatasource 特有的初始化 + self.graph_dict = self._load_graph_dict() +``` + +## 特殊参数处理 + +### 1. TgHydroDatasource 没有独有参数 +- TgHydroDatasource 的构造函数参数与父类 SelfMadeHydroDataset 完全相同 +- 所有参数都通过 `super().__init__()` 传递给父类处理 +- 子类只是在父类初始化完成后，添加了图网络结构的加载 + +### 2. 如果子类有独有参数的处理方式 + +假设 TgHydroDatasource 有独有参数，处理方式如下： + +```python +def __init__(self, data_path, download=False, time_unit=None, dataset_name=None, + graph_config=None, **kwargs): # graph_config 是子类独有参数 + """ + Parameters + ---------- + graph_config : dict, optional + 图网络配置参数（子类独有） + """ + # 处理子类独有参数 + self.graph_config = graph_config or {} + + # 调用父类初始化（不传递子类独有参数） + super().__init__(data_path, download, time_unit, dataset_name, **kwargs) + + # 子类特有的初始化 + self.graph_dict = self._load_graph_dict() +``` + +## 数据目录结构要求 + +TgHydroDatasource 期望的数据目录结构： + +``` +data_path/ +├── attributes/ # 流域属性数据（父类要求） +│ └── attributes.csv +├── timeseries/ # 时间序列数据（父类要求） +│ ├── 1D/ # 日数据 +│ ├── 3h/ # 3小时数据 +│ └── *_units_info.json +├── shapes/ # 流域形状文件（父类要求） +│ └── *.shp +└── tggnn/ # TG-GNN专用数据（子类要求） + ├── lstmpred.nc # LSTM预测数据 + └── graph_dict.json # 图网络结构 +``` + +## 使用示例 + +### 1. 基本使用 +```python +from hydrodatasource.reader.data_source import TgHydroDatasource + +# 最简单的使用方式 +tg_datasource = TgHydroDatasource(data_path="/path/to/data") +``` + +### 2. 完整参数使用 +```python +tg_datasource = TgHydroDatasource( + data_path="/path/to/data", + download=False, + time_unit=["1D", "3h"], + dataset_name="my_tg_dataset", + version="v1.0" +) +``` + +### 3. S3路径使用 +```python +tg_datasource = TgHydroDatasource( + data_path="s3://my-bucket/tg-data", + time_unit=["1D"], + dataset_name="s3_tg_dataset" +) +``` + +## 注意事项 + +1. **time_unit 限制**：只支持 ["1h", "3h", "1D", "8D"] 四种时间单位 +2. **数据路径**：支持本地路径和S3路径，S3路径需要以 "s3://" 开头 +3. **图网络文件**：必须存在 `tggnn/graph_dict.json` 文件，否则初始化会失败 +4. **继承关系**：所有父类的方法和属性都可以使用 +5. **参数传递**：kwargs 中的参数会传递给父类，可以用于扩展功能 \ No newline at end of file diff --git "a/TgHydroDatasource_\350\276\223\345\207\272\346\216\245\345\217\243\350\257\264\346\230\216.md" "b/TgHydroDatasource_\350\276\223\345\207\272\346\216\245\345\217\243\350\257\264\346\230\216.md" new file mode 100644 index 0000000..21fcd30 --- /dev/null +++ "b/TgHydroDatasource_\350\276\223\345\207\272\346\216\245\345\217\243\350\257\264\346\230\216.md" @@ -0,0 +1,225 @@ +# TgHydroDatasource 类输出接口说明 + +## 概述 + +`TgHydroDatasource` 类继承自 `SelfMadeHydroDataset`，提供了丰富的数据读取接口，主要用于图神经网络相关的水文建模任务。该类的输出接口可以分为以下几个类别： + +## 1. 基础数据读取接口（继承自父类） + +### 1.1 流域ID读取 +```python +def read_object_ids(self, object_params=None) -> np.array +``` +- **功能**: 读取所有可用的流域ID +- **返回**: numpy数组，包含所有流域ID +- **示例**: `basin_ids = tg_datasource.read_object_ids()` + +### 1.2 时间序列数据读取 +```python +def read_timeseries(self, object_ids=None, t_range_list=None, relevant_cols=None, **kwargs) -> dict +``` +- **功能**: 读取原始时间序列数据 +- **参数**: + - `object_ids`: 流域ID列表 + - `t_range_list`: 时间范围 `[start_time, end_time]` + - `relevant_cols`: 变量列表 + - `time_units`: 时间单位（如 `["1D", "3h"]`） +- **返回**: 字典，键为时间单位，值为numpy数组 `(basin_num, time_num, var_num)` + +### 1.3 属性数据读取 +```python +def read_attributes(self, object_ids=None, constant_cols=None, **kwargs) -> np.array +``` +- **功能**: 读取流域静态属性数据 +- **参数**: + - `object_ids`: 流域ID列表 + - `constant_cols`: 属性列名列表 +- **返回**: numpy数组 `(basin_num, attr_num)` + +### 1.4 xarray格式数据读取 +```python +def read_ts_xrdataset(self, gage_id_lst=None, t_range=None, var_lst=None, **kwargs) -> dict +``` +- **功能**: 读取xarray格式的时间序列数据 +- **返回**: 字典，键为时间单位，值为xarray.Dataset + +```python +def read_attr_xrdataset(self, gage_id_lst=None, var_lst=None, **kwargs) -> xr.Dataset +``` +- **功能**: 读取xarray格式的属性数据 +- **返回**: xarray.Dataset + +### 1.5 特定数据读取 +```python +def read_area(self, gage_id_lst=None) -> xr.Dataset +``` +- **功能**: 读取流域面积数据 +- **返回**: xarray.Dataset，包含面积信息 + +```python +def read_mean_prcp(self, gage_id_lst=None, unit="mm/d") -> xr.Dataset +``` +- **功能**: 读取平均降水数据 +- **返回**: xarray.Dataset，包含降水信息 + +## 2. TgHydroDatasource 特有接口 + +### 2.1 LSTM预测数据读取 +```python +def read_lstm_predictions(self, object_ids=None, t_range_list=None, **kwargs) -> xr.Dataset +``` +- **功能**: 读取LSTM预测数据 +- **参数**: + - `object_ids`: 流域ID列表 + - `t_range_list`: 时间范围 `[start_time, end_time]` +- **返回**: xarray.Dataset，包含LSTM预测结果 +- **数据来源**: `tggnn/lstmpred.nc` 文件 + +### 2.2 时间序列与LSTM数据联合读取 +```python +def read_timeseries_with_lstm(self, object_ids=None, t_range=None, var_lst=None, time_units=None) -> tuple +``` +- **功能**: 同时读取时间序列数据和LSTM预测数据 +- **参数**: + - `object_ids`: 流域ID列表 + - `t_range`: 时间范围 `[start_time, end_time]` + - `var_lst`: 变量列表（默认: `['total_precipitation_hourly', 'streamflow']`） + - `time_units`: 时间单位 +- **返回**: 元组 `(timeseries_data, lstm_data)`，两个都是xarray.Dataset + +### 2.3 节点属性数据读取 +```python +def read_node_attributes(self, object_ids=None, selected_attrs=None) -> xr.Dataset +``` +- **功能**: 读取节点静态属性数据（简化版，无PyTorch转换） +- **参数**: + - `object_ids`: 流域ID列表 + - `selected_attrs`: 属性列表（默认: `['ele_mt_sav', 'area', 'p_mean']`） +- **返回**: xarray.Dataset，包含筛选后的属性数据 + +### 2.4 图网络结构生成 +```python +def gen_nx_graph(self, basin_id, object_ids=None) -> tuple +``` +- **功能**: 生成NetworkX图对象，确保节点索引与数据一致 +- **参数**: + - `basin_id`: 流域ID（用于查找图结构） + - `object_ids`: 对象ID列表（可选） +- **返回**: 元组 `(NetworkX图对象, 边列表, 节点映射字典, 有效对象ID列表)` +- **数据来源**: `tggnn/graph_dict.json` 文件 + +### 2.5 上游节点查询 +```python +def get_upstream_nodes(self, basin_id, target_node, object_ids=None) -> list +``` +- **功能**: 获取指定节点的直接上游节点 +- **参数**: + - `basin_id`: 流域ID + - `target_node`: 目标节点名称 + - `object_ids`: 所有节点名称列表（可选） +- **返回**: 直接上游节点的索引列表 + +## 3. 元数据和信息接口 + +### 3.1 数据源信息 +```python +def get_name(self) -> str +``` +- **功能**: 返回数据源名称 +- **返回**: `"TgHydroDatasource"` + +```python +def set_data_source_describe(self) -> collections.OrderedDict +``` +- **功能**: 设置数据源描述，包含所有数据路径 +- **返回**: 有序字典，包含数据路径信息 + +### 3.2 列信息获取 +```python +def get_attributes_cols(self) -> np.array +``` +- **功能**: 获取属性数据的列名 +- **返回**: numpy数组，包含所有属性列名 + +```python +def get_timeseries_cols(self) -> dict +``` +- **功能**: 获取时间序列数据的列名 +- **返回**: 字典，键为时间单位，值为变量列表 + +## 4. 数据格式说明 + +### 4.1 返回数据类型 +- **numpy.array**: 原始数据读取方法返回 +- **xarray.Dataset**: 现代化数据读取方法返回，包含坐标、属性和元数据 +- **dict**: 多时间单位数据返回 +- **tuple**: 多种数据类型组合返回 + +### 4.2 坐标系统 +- **basin**: 流域维度，使用流域ID作为坐标 +- **time**: 时间维度，使用pandas时间索引 +- **变量维度**: 根据具体数据类型而定 + +### 4.3 数据筛选 +所有读取方法都支持： +- 按流域ID筛选（`object_ids`参数） +- 按时间范围筛选（`t_range`或`t_range_list`参数） +- 按变量筛选（`var_lst`或`selected_attrs`参数） + +## 5. 使用建议 + +### 5.1 推荐使用的接口 +- **现代化接口**: 优先使用返回xarray.Dataset的方法 +- **联合读取**: 使用`read_timeseries_with_lstm`同时获取多种数据 +- **图网络**: 使用`gen_nx_graph`获取完整的图结构信息 + +### 5.2 性能考虑 +- 大数据量时使用缓存机制（`cache_xrdataset`） +- 按需读取，避免加载不必要的数据 +- 合理设置时间范围和变量列表 + +### 5.3 错误处理 +- 所有方法都包含数据验证和错误提示 +- 缺失数据会给出警告信息 +- 支持部分数据缺失的情况下继续处理 + +## 6. 示例用法 + +```python +from hydrodatasource.reader.data_source import TgHydroDatasource + +# 初始化 +tg_datasource = TgHydroDatasource(data_path="/path/to/data") + +# 获取基础信息 +basin_ids = tg_datasource.read_object_ids() +print(f"可用流域数量: {len(basin_ids)}") + +# 读取时间序列和LSTM数据 +timeseries_data, lstm_data = tg_datasource.read_timeseries_with_lstm( + object_ids=basin_ids[:5], + t_range=['2020-01-01', '2020-12-31'], + var_lst=['streamflow', 'total_precipitation_hourly'] +) + +# 读取属性数据 +attr_data = tg_datasource.read_node_attributes( + object_ids=basin_ids[:5], + selected_attrs=['area', 'ele_mt_sav'] +) + +# 生成图网络 +graph, edges, node_mapping, valid_ids = tg_datasource.gen_nx_graph( + basin_id='sanxia_60703800', + object_ids=basin_ids[:10] +) + +# 查询上游节点 +upstream_nodes = tg_datasource.get_upstream_nodes( + basin_id='sanxia_60703800', + target_node='sanxia_60703780', + object_ids=valid_ids +) +``` + +这个接口设计提供了从基础数据读取到高级图网络分析的完整功能，特别适合图神经网络相关的水文建模任务。 \ No newline at end of file diff --git a/example_tg_usage.py b/example_tg_usage.py new file mode 100644 index 0000000..623258f --- /dev/null +++ b/example_tg_usage.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +简化后的TgHydroDatasource类使用示例 + +该示例展示了如何使用简化后的TgHydroDatasource类进行数据读取， +不涉及torch转换，保持数据的原始xarray格式。 +""" + +import os +import sys +from pathlib import Path + +# 添加项目路径 +project_root = Path(__file__).parent +sys.path.insert(0, str(project_root)) + +from hydrodatasource.reader.data_source import TgHydroDatasource + + +def example_usage(): + """展示TgHydroDatasource的基本使用方法""" + + # 假设数据路径 + data_path = "/path/to/your/tg_hydro_data" + + # 初始化TgHydroDatasource + tg_datasource = TgHydroDatasource( + data_path=data_path, + time_unit=['1D'], + dataset_name="example_dataset" + ) + + print(f"数据源名称: {tg_datasource.get_name()}") + print(f"数据路径: {tg_datasource.data_source_dir}") + + # 1. 读取可用的流域ID + available_basins = tg_datasource.read_object_ids() + print(f"可用流域数量: {len(available_basins)}") + print(f"前5个流域ID: {available_basins[:5]}") + + # 2. 读取时间序列数据和LSTM预测数据 + selected_basins = available_basins[:3] # 选择前3个流域 + time_range = ['2020-01-01', '2020-12-31'] + variables = ['total_precipitation_hourly', 'streamflow'] + + timeseries_data, lstm_data = tg_datasource.read_timeseries_with_lstm( + object_ids=selected_basins, + t_range=time_range, + var_lst=variables + ) + + print(f"\n时间序列数据:") + print(f" 数据类型: {type(timeseries_data)}") + print(f" 时间单位: {list(timeseries_data.keys())}") + for time_unit, data in timeseries_data.items(): + print(f" {time_unit} 数据形状: {data.dims}") + print(f" {time_unit} 变量: {list(data.data_vars)}") + + print(f"\nLSTM预测数据:") + print(f" 数据类型: {type(lstm_data)}") + print(f" 数据形状: {lstm_data.dims}") + print(f" 变量: {list(lstm_data.data_vars)}") + + # 3. 读取属性数据 + selected_attrs = ['ele_mt_sav', 'area', 'p_mean'] + attr_data = tg_datasource.read_node_attributes( + object_ids=selected_basins, + selected_attrs=selected_attrs + ) + + print(f"\n属性数据:") + print(f" 数据类型: {type(attr_data)}") + print(f" 数据形状: {attr_data.dims}") + print(f" 变量: {list(attr_data.data_vars)}") + + # 4. 生成图网络结构 + basin_id = "test_basin" # 根据实际情况修改 + try: + dg, edges, node_mapping, valid_object_ids = tg_datasource.gen_nx_graph( + basin_id=basin_id, + object_ids=selected_basins + ) + + print(f"\n图网络结构:") + print(f" 节点数量: {len(dg.nodes)}") + print(f" 边数量: {len(edges)}") + print(f" 节点映射: {node_mapping}") + print(f" 有效对象ID: {valid_object_ids}") + + # 5. 获取上游节点 + if valid_object_ids: + target_node = valid_object_ids[0] + upstream_nodes = tg_datasource.get_upstream_nodes( + basin_id=basin_id, + target_node=target_node, + object_ids=selected_basins + ) + print(f" {target_node} 的上游节点索引: {upstream_nodes}") + + except ValueError as e: + print(f"\n图网络结构错误: {e}") + print("请检查graph_dict.json文件是否存在且包含正确的流域ID") + + # 6. 使用父类方法读取数据（展示继承功能） + print(f"\n使用父类方法读取数据:") + + # 读取时间序列数据 + ts_data = tg_datasource.read_ts_xrdataset( + gage_id_lst=selected_basins, + t_range=time_range, + var_lst=['streamflow'], + time_units=['1D'] + ) + print(f" 时间序列数据 (父类方法): {type(ts_data)}") + + # 读取属性数据 + attr_data_parent = tg_datasource.read_attr_xrdataset( + var_lst=['area'] + ) + print(f" 属性数据 (父类方法): {type(attr_data_parent)}") + + +def data_processing_example(): + """展示如何处理读取的数据""" + + print("\n" + "="*50) + print("数据处理示例") + print("="*50) + + # 这里展示如何处理从TgHydroDatasource读取的xarray数据 + # 用户可以根据需要进行各种数据处理和分析 + + print("1. 数据筛选和切片") + print(" - 使用 .sel() 方法按坐标筛选") + print(" - 使用 .isel() 方法按索引筛选") + print(" - 使用 .where() 方法按条件筛选") + + print("\n2. 数据转换") + print(" - 使用 .values 获取numpy数组") + print(" - 使用 .to_pandas() 转换为pandas") + print(" - 使用 .to_netcdf() 保存为NetCDF文件") + + print("\n3. 数据分析") + print(" - 使用 .mean(), .std(), .max(), .min() 等统计方法") + print(" - 使用 .groupby() 进行分组分析") + print(" - 使用 .resample() 进行时间重采样") + + print("\n4. 如果需要torch张量，可以手动转换:") + print(" import torch") + print(" tensor = torch.from_numpy(data.values).float()") + + +if __name__ == "__main__": + print("TgHydroDatasource 简化版使用示例") + print("="*50) + + try: + example_usage() + data_processing_example() + + print("\n" + "="*50) + print("示例完成！") + print("="*50) + + except Exception as e: + print(f"运行示例时出错: {e}") + print("请确保数据路径正确且数据文件存在") \ No newline at end of file diff --git a/hydrodatasource/reader/data_source.py b/hydrodatasource/reader/data_source.py index 2e00ae1..80de764 100644 --- a/hydrodatasource/reader/data_source.py +++ b/hydrodatasource/reader/data_source.py @@ -9,6 +9,7 @@ from abc import ABC from tqdm import tqdm import polars as pl +import networkx as nx from hydroutils import hydro_file from hydroutils.hydro_time import generate_start0101_time_range @@ -1708,3 +1709,355 @@ def get_stations_by_basin(self, basin_id: str) -> list: return self.basin_station_mapping[ self.basin_station_mapping["basin_id"] == basin_id ]["station_id"].unique().tolist() + +class TgHydroDatasource(SelfMadeHydroDataset): + """ + TG流域数据集 - 继承自SelfMadeHydroDataset，添加LSTM预测数据和图网络结构支持 + + 该类在标准的水文数据集基础上，增加了对LSTM预测数据和图网络结构的支持， + 主要用于图神经网络相关的水文建模任务。 + + 数据目录结构: + - attributes/ # 流域属性数据 + - timeseries/ # 时间序列数据 + - shapes/ # 流域形状文件 + - tggnn/ # TG-GNN专用数据 + ├── lstmpred.nc # LSTM预测数据 + └── graph_dict.json # 图网络结构 + """ + + def __init__(self, data_path, download=False, time_unit=None, dataset_name=None, **kwargs): + """ + 初始化TG流域数据集 + + Parameters + ---------- + data_path : str + 数据根目录路径 + download : bool, optional + 是否下载数据, by default False + time_unit : list, optional + 时间单位列表, by default None + dataset_name : str, optional + 数据集名称, by default None + **kwargs : dict + 其他参数 + """ + # 调用父类初始化 + super().__init__(data_path, download, time_unit, dataset_name, **kwargs) + + # 加载图网络结构 + self.graph_dict = self._load_graph_dict() + + def get_name(self): + """返回数据源名称""" + return "TgHydroDatasource" + + def set_data_source_describe(self): + """ + 设置数据源描述，在父类基础上添加TGGNN相关路径 + + Returns + ------- + collections.OrderedDict + 包含所有数据路径的有序字典 + """ + # 获取父类的数据源描述 + data_source_desc = super().set_data_source_describe() + + # 添加TGGNN相关路径 + data_root_dir = self.data_source_dir + tggnn_dir = os.path.join(data_root_dir, "tggnn") + lstm_pred_file = os.path.join(tggnn_dir, "lstmpred.nc") + graph_dict_file = os.path.join(tggnn_dir, "graph_dict.json") + + # 更新数据源描述 + data_source_desc.update({ + "TGGNN_DIR": tggnn_dir, + "LSTM_PRED_FILE": lstm_pred_file, + "GRAPH_DICT_FILE": graph_dict_file, + }) + + return data_source_desc + + def _load_graph_dict(self): + """ + 从JSON文件加载图网络结构 + + Returns + ------- + dict + 图网络结构字典 + """ + graph_dict_file = self.data_source_description["GRAPH_DICT_FILE"] + + try: + if "s3://" in graph_dict_file: + with conf.FS.open(graph_dict_file, mode="rb") as f: + graph_dict = json.load(f) + else: + with open(graph_dict_file, 'r', encoding='utf-8') as f: + graph_dict = json.load(f) + except FileNotFoundError: + # 如果文件不存在，返回默认的图网络结构 + print(f"警告：图网络结构文件 {graph_dict_file} 不存在，使用默认结构") + graph_dict = self._get_default_graph_dict() + # 保存默认结构到文件 + self._save_graph_dict(graph_dict) + + return graph_dict + + def _get_default_graph_dict(self): + """ + 获取默认的图网络结构 + + Returns + ------- + dict + 默认图网络结构 + """ + return { + 'sanxia_60703800': [ + ['sanxia_60718300', 'sanxia_60703780', 'sanxia_60703800'], + ['sanxia_60716700', 'sanxia_60715400', 'sanxia_60715955', 'sanxia_60703780', 'sanxia_60703800'], + ['sanxia_60716250', 'sanxia_60715000', 'sanxia_60715400', 'sanxia_60715955', 'sanxia_60703780', 'sanxia_60703800'], + ['sanxia_60701001', 'sanxia_60715000', 'sanxia_60715400', 'sanxia_60715955', 'sanxia_60703780', 'sanxia_60703800'], + ['sanxia_60717050', 'sanxia_60715400', 'sanxia_60715955', 'sanxia_60703780', 'sanxia_60703800'], + ['sanxia_60710650', 'sanxia_60703700', 'sanxia_60703780', 'sanxia_60703800'], + ['sanxia_60706825', 'sanxia_60700001', 'sanxia_60700002', 'sanxia_60703700', 'sanxia_60703780', 'sanxia_60703800'], + ['sanxia_60701750', 'sanxia_60700002', 'sanxia_60703700', 'sanxia_60703780', 'sanxia_60703800'], + ['sanxia_60709859', 'sanxia_60703700', 'sanxia_60703780', 'sanxia_60703800'], + ['sanxia_60713800', 'sanxia_60712000', 'sanxia_60703780', 'sanxia_60703800'], + ['sanxia_60713730', 'sanxia_60711800', 'sanxia_60712000', 'sanxia_60703780', 'sanxia_60703800'], + ['sanxia_60713630', 'sanxia_60711800', 'sanxia_60712000', 'sanxia_60703780', 'sanxia_60703800'], + ['sanxia_60713400', 'sanxia_60706011', 'sanxia_60713170', 'sanxia_60711800', 'sanxia_60712000', 'sanxia_60703780', 'sanxia_60703800'], + ['sanxia_60713270', 'sanxia_60713170', 'sanxia_60711800', 'sanxia_60712000', 'sanxia_60703780', 'sanxia_60703800'], + ['sanxia_60712900', 'sanxia_60711800', 'sanxia_60712000', 'sanxia_60703780', 'sanxia_60703800'], + ['sanxia_60701101', 'sanxia_60711800', 'sanxia_60712000', 'sanxia_60703780', 'sanxia_60703800'], + ['sanxia_60711600', 'sanxia_60711800', 'sanxia_60712000', 'sanxia_60703780', 'sanxia_60703800'] + ] + } + + def _save_graph_dict(self, graph_dict): + """ + 保存图网络结构到JSON文件 + + Parameters + ---------- + graph_dict : dict + 图网络结构字典 + """ + graph_dict_file = self.data_source_description["GRAPH_DICT_FILE"] + + # 确保目录存在 + os.makedirs(os.path.dirname(graph_dict_file), exist_ok=True) + + if "s3://" in graph_dict_file: + with conf.FS.open(graph_dict_file, mode="w") as f: + json.dump(graph_dict, f, ensure_ascii=False, indent=2) + else: + with open(graph_dict_file, 'w', encoding='utf-8') as f: + json.dump(graph_dict, f, ensure_ascii=False, indent=2) + + def read_lstm_predictions(self, object_ids=None, t_range_list=None, **kwargs): + """ + 读取LSTM预测数据 + + Parameters + ---------- + object_ids : list, optional + 流域ID列表, by default None + t_range_list : list, optional + 时间范围列表, by default None + **kwargs : dict + 其他参数 + + Returns + ------- + xr.Dataset + LSTM预测数据集 + """ + lstm_pred_file = self.data_source_description["LSTM_PRED_FILE"] + + if "s3://" in lstm_pred_file: + with conf.FS.open(lstm_pred_file, mode="rb") as f: + lstm_data = xr.open_dataset(f) + else: + lstm_data = xr.open_dataset(lstm_pred_file) + + # 根据参数筛选数据 + if object_ids is not None: + lstm_data = lstm_data.sel(basin=object_ids) + + if t_range_list is not None: + start_time = pd.to_datetime(t_range_list[0]) + end_time = pd.to_datetime(t_range_list[1]) + lstm_data = lstm_data.sel(time=slice(start_time, end_time)) + + return lstm_data + + def read_timeseries_with_lstm(self, object_ids=None, t_range=None, var_lst=None, time_units=None): + """ + 读取时间序列数据和LSTM预测数据 + + Parameters + ---------- + object_ids : list, optional + 对象ID列表（流域名称）, by default None + t_range : list, optional + 时间范围，格式为[start_time, end_time], by default None + var_lst : list, optional + 变量列表, by default None + time_units : list, optional + 时间单位，默认使用类的time_unit属性, by default None + + Returns + ------- + tuple + (时间序列数据xarray.Dataset, LSTM预测数据xarray.Dataset) + """ + if time_units is None: + time_units = self.time_unit + + var_lst_tggnn = ['total_precipitation_hourly', 'discharge', 'streamflow'] + + # 使用父类方法读取时间序列数据 + timeseries_data = self.read_ts_xrdataset( + gage_id_lst=object_ids, + t_range=t_range, + var_lst=var_lst_tggnn, + time_units=time_units + ) + + # 读取LSTM预测数据 + lstm_data = self.read_lstm_predictions( + object_ids=object_ids, + t_range_list=t_range + ) + + return timeseries_data, lstm_data + + def read_node_attributes(self, object_ids=None, selected_attrs=None): + """ + 读取节点静态属性数据 + + Parameters + ---------- + object_ids : array-like, optional + 流域ID列表, by default None + selected_attrs : list, optional + 选择的属性列表, by default None + + Returns + ------- + xr.Dataset + 属性数据集 + """ + if selected_attrs is None: + selected_attrs = ['ele_mt_sav', 'area', 'p_mean'] + + # 使用父类方法读取属性数据 + attr_data = self.read_attr_xrdataset(var_lst=selected_attrs) + + # 如果指定了object_ids，则筛选数据 + if object_ids is not None: + # 确保object_ids中的所有ID都存在于数据中 + available_basins = attr_data.coords['basin'].values + valid_ids = [basin_id for basin_id in object_ids if basin_id in available_basins] + if len(valid_ids) != len(object_ids): + missing_ids = set(object_ids) - set(valid_ids) + print(f"警告：以下流域ID在属性数据中不存在: {missing_ids}") + + if valid_ids: + attr_data = attr_data.sel(basin=valid_ids) + + return attr_data + + def gen_nx_graph(self, basin_id, object_ids=None): + """ + 生成网络图，确保节点索引与数据中basin顺序一致 + + Parameters + ---------- + basin_id : str + 流域ID + object_ids : array-like, optional + 对象ID列表，如果为None则使用所有可用的流域ID, by default None + + Returns + ------- + tuple + (NetworkX图对象, 边列表, 节点映射字典, 有效的对象ID列表) + """ + basin_path_lists = self.graph_dict.get(basin_id) + if basin_path_lists is None: + raise ValueError(f"未找到流域 {basin_id} 的图网络结构") + + # 构建有向图 + dg = nx.DiGraph() + for path in basin_path_lists: + nx.add_path(dg, path) + + # 获取图中所有节点 + graph_nodes = set(dg.nodes) + + # 如果没有指定object_ids，使用图中所有节点 + if object_ids is None: + # 从数据中获取可用的流域ID + available_basins = self.read_object_ids() + # 取图中节点与可用数据的交集 + valid_object_ids = [basin_id for basin_id in available_basins if basin_id in graph_nodes] + else: + # 检查指定的object_ids是否在图中存在 + object_ids_list = list(object_ids) + valid_object_ids = [basin_id for basin_id in object_ids_list if basin_id in graph_nodes] + + # 报告不在图中的节点 + missing_in_graph = set(object_ids_list) - graph_nodes + if missing_in_graph: + print(f"警告：以下流域ID在图网络结构中不存在: {missing_in_graph}") + + # 创建节点映射（节点名称到索引的映射） + node_mapping = {node_id: idx for idx, node_id in enumerate(valid_object_ids)} + + # 生成边列表（使用索引） + edges = [] + for u, v in dg.edges: + if u in node_mapping and v in node_mapping: + edges.append((node_mapping[u], node_mapping[v])) + + return dg, edges, node_mapping, valid_object_ids + + def classify_nodes(dg, basin_names): + """ + 分类节点为源头节点和汇流节点 + + Args: + dg: NetworkX有向图 + basin_names: 节点名称列表 + + Returns: + source_nodes: 源头节点列表（没有入度的节点） + confluence_nodes: 汇流节点列表（有入度的节点） + node_mask: 布尔掩码，True表示汇流节点，False表示源头节点 + """ + source_nodes = [] + confluence_nodes = [] + + for node in basin_names: + # 检查节点的入度 + in_degree = dg.in_degree(node) + if in_degree == 0: + source_nodes.append(node) + else: + confluence_nodes.append(node) + + # 创建节点掩码：True表示汇流节点（需要训练），False表示源头节点（不训练） + node_mask = [] + for node in basin_names: + node_mask.append(node in confluence_nodes) + + print(f"源头节点 ({len(source_nodes)}个): {source_nodes}") + print(f"汇流节点 ({len(confluence_nodes)}个): {confluence_nodes}") + + return source_nodes, confluence_nodes, node_mask diff --git "a/read_lstm_predictions_\345\217\202\346\225\260\350\257\264\346\230\216.md" "b/read_lstm_predictions_\345\217\202\346\225\260\350\257\264\346\230\216.md" new file mode 100644 index 0000000..3215927 --- /dev/null +++ "b/read_lstm_predictions_\345\217\202\346\225\260\350\257\264\346\230\216.md" @@ -0,0 +1,195 @@ +# read_lstm_predictions 方法参数传递说明 + +## 方法签名 + +```python +def read_lstm_predictions(self, object_ids=None, t_range_list=None, **kwargs): +``` + +## 参数传递方式 + +### 1. 直接调用方式 + +用户可以直接调用`read_lstm_predictions`方法： + +```python +from hydrodatasource.reader.data_source import TgHydroDatasource + +# 初始化数据源 +tg_datasource = TgHydroDatasource(data_path="/path/to/data") + +# 直接调用read_lstm_predictions +lstm_data = tg_datasource.read_lstm_predictions( + object_ids=['basin_1', 'basin_2', 'basin_3'], + t_range_list=['2020-01-01', '2020-12-31'] +) +``` + +### 2. 通过read_timeseries_with_lstm间接调用 + +更常见的使用方式是通过`read_timeseries_with_lstm`方法间接调用： + +```python +# 通过read_timeseries_with_lstm调用 +timeseries_data, lstm_data = tg_datasource.read_timeseries_with_lstm( + object_ids=['basin_1', 'basin_2', 'basin_3'], # 这个参数会传递给read_lstm_predictions + t_range=['2020-01-01', '2020-12-31'], # 这个参数会传递给read_lstm_predictions的t_range_list + var_lst=['streamflow', 'precipitation'] +) +``` + +## 参数详细说明 + +### object_ids 参数 + +| 属性 | 说明 | +|------|------| +| **类型** | `list` 或 `None` | +| **默认值** | `None` | +| **作用** | 指定要读取的流域ID列表 | +| **传递方式** | 直接传递或通过`read_timeseries_with_lstm`的`object_ids`参数传递 | + +**使用示例：** +```python +# 方式1：直接传递 +lstm_data = tg_datasource.read_lstm_predictions( + object_ids=['basin_001', 'basin_002', 'basin_003'] +) + +# 方式2：通过read_timeseries_with_lstm传递 +timeseries_data, lstm_data = tg_datasource.read_timeseries_with_lstm( + object_ids=['basin_001', 'basin_002', 'basin_003'] # 会传递给read_lstm_predictions +) +``` + +**内部处理逻辑：** +```python +# 在read_lstm_predictions方法内部 +if object_ids is not None: + lstm_data = lstm_data.sel(basin=object_ids) # 使用xarray的sel方法筛选流域 +``` + +### t_range_list 参数 + +| 属性 | 说明 | +|------|------| +| **类型** | `list` 或 `None` | +| **默认值** | `None` | +| **格式** | `[start_time, end_time]` | +| **作用** | 指定要读取的时间范围 | +| **传递方式** | 直接传递或通过`read_timeseries_with_lstm`的`t_range`参数传递 | + +**使用示例：** +```python +# 方式1：直接传递 +lstm_data = tg_datasource.read_lstm_predictions( + t_range_list=['2020-01-01', '2020-12-31'] +) + +# 方式2：通过read_timeseries_with_lstm传递 +timeseries_data, lstm_data = tg_datasource.read_timeseries_with_lstm( + t_range=['2020-01-01', '2020-12-31'] # 会传递给read_lstm_predictions的t_range_list +) +``` + +**内部处理逻辑：** +```python +# 在read_lstm_predictions方法内部 +if t_range_list is not None: + start_time = pd.to_datetime(t_range_list[0]) + end_time = pd.to_datetime(t_range_list[1]) + lstm_data = lstm_data.sel(time=slice(start_time, end_time)) # 使用xarray的时间切片 +``` + +## 参数传递流程图 + +``` +用户调用 + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 方式1: 直接调用 │ +│ tg_datasource.read_lstm_predictions( │ +│ object_ids=['basin_1', 'basin_2'], │ +│ t_range_list=['2020-01-01', '2020-12-31'] │ +│ ) │ +└─────────────────────────────────────────────────────────────┘ + ↓ +read_lstm_predictions方法接收参数 + ↓ +处理和筛选数据 + +┌─────────────────────────────────────────────────────────────┐ +│ 方式2: 通过read_timeseries_with_lstm间接调用 │ +│ tg_datasource.read_timeseries_with_lstm( │ +│ object_ids=['basin_1', 'basin_2'], │ +│ t_range=['2020-01-01', '2020-12-31'], │ +│ var_lst=['streamflow'] │ +│ ) │ +└─────────────────────────────────────────────────────────────┘ + ↓ +read_timeseries_with_lstm方法内部调用 + ↓ +self.read_lstm_predictions( + object_ids=object_ids, # 直接传递 + t_range_list=t_range # 参数名映射：t_range → t_range_list +) + ↓ +read_lstm_predictions方法接收参数 + ↓ +处理和筛选数据 +``` + +## 参数验证和错误处理 + +### object_ids 验证 +- 如果传入的`object_ids`在数据集中不存在，xarray会抛出`KeyError` +- 建议先使用`tg_datasource.read_object_ids()`获取可用的流域ID + +### t_range_list 验证 +- 时间格式会通过`pd.to_datetime()`自动解析 +- 支持多种时间格式：'2020-01-01', '2020/01/01', '20200101'等 +- 如果时间范围超出数据集范围，会返回空的时间切片 + +## 完整使用示例 + +```python +from hydrodatasource.reader.data_source import TgHydroDatasource + +# 初始化 +tg_datasource = TgHydroDatasource(data_path="/path/to/data") + +# 获取可用的流域ID +available_basins = tg_datasource.read_object_ids() +print(f"可用流域: {available_basins[:5]}") + +# 选择要处理的流域和时间范围 +selected_basins = available_basins[:3] +time_range = ['2020-01-01', '2020-12-31'] + +# 方式1：直接调用read_lstm_predictions +lstm_data_direct = tg_datasource.read_lstm_predictions( + object_ids=selected_basins, + t_range_list=time_range +) + +# 方式2：通过read_timeseries_with_lstm调用 +timeseries_data, lstm_data_indirect = tg_datasource.read_timeseries_with_lstm( + object_ids=selected_basins, + t_range=time_range, + var_lst=['streamflow', 'precipitation'] +) + +# 两种方式得到的lstm_data应该是相同的 +print(f"直接调用结果: {lstm_data_direct.dims}") +print(f"间接调用结果: {lstm_data_indirect.dims}") +``` + +## 注意事项 + +1. **参数名映射**：`read_timeseries_with_lstm`中的`t_range`参数会映射为`read_lstm_predictions`中的`t_range_list`参数 + +2. **数据筛选顺序**：先按流域筛选，再按时间筛选 + +3. **性能考虑**：如果数据集很大，建议先筛选时间范围再筛选流域，以提高性能 + +4. **错误处理**：建议在调用前验证参数的有效性，避免运行时错误 \ No newline at end of file diff --git a/test_tg_hydro_datasource.py b/test_tg_hydro_datasource.py new file mode 100644 index 0000000..3e8563f --- /dev/null +++ b/test_tg_hydro_datasource.py @@ -0,0 +1,334 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +测试TgHydroDatasource类的功能 +""" + +import os +import sys +import json +import numpy as np +import pandas as pd +import xarray as xr +import geopandas as gpd +from shapely.geometry import Polygon +from pathlib import Path + +# 添加项目路径到sys.path +project_root = Path(__file__).parent +sys.path.insert(0, str(project_root)) + +from hydrodatasource.reader.data_source import TgHydroDatasource + + +def create_test_data_structure(test_data_dir): + """创建测试数据目录结构""" + print("创建测试数据目录结构...") + + # 创建目录结构 + dirs_to_create = [ + "attributes", + "timeseries/1D", + "shapes", + "tggnn" + ] + + for dir_name in dirs_to_create: + dir_path = os.path.join(test_data_dir, dir_name) + os.makedirs(dir_path, exist_ok=True) + print(f"创建目录: {dir_path}") + + +def create_test_graph_dict(test_data_dir): + """创建测试用的graph_dict.json文件""" + print("创建测试graph_dict.json文件...") + + graph_dict = { + 'test_basin': [ + ['station_1', 'station_2', 'station_3'], + ['station_4', 'station_2', 'station_3'], + ['station_5', 'station_6', 'station_3'] + ] + } + + graph_dict_file = os.path.join(test_data_dir, "tggnn", "graph_dict.json") + with open(graph_dict_file, 'w', encoding='utf-8') as f: + json.dump(graph_dict, f, ensure_ascii=False, indent=2) + + print(f"创建文件: {graph_dict_file}") + return graph_dict + + +def create_test_lstm_predictions(test_data_dir): + """创建测试用的lstmpred.nc文件""" + print("创建测试lstmpred.nc文件...") + + # 创建测试数据 + time_range = pd.date_range('2020-01-01', '2020-01-10', freq='H') + basin_names = ['station_1', 'station_2', 'station_3', 'station_4', 'station_5', 'station_6'] + + # 生成随机流量数据 + np.random.seed(42) + streamflow_data = np.random.rand(len(basin_names), len(time_range)) * 100 + + # 创建xarray Dataset + lstm_dataset = xr.Dataset({ + 'streamflow': (['basin', 'time'], streamflow_data) + }, coords={ + 'basin': basin_names, + 'time': time_range + }) + + lstm_pred_file = os.path.join(test_data_dir, "tggnn", "lstmpred.nc") + lstm_dataset.to_netcdf(lstm_pred_file) + + print(f"创建文件: {lstm_pred_file}") + return lstm_dataset + + +def create_test_timeseries(test_data_dir): + """创建测试用的时间序列数据""" + print("创建测试时间序列数据...") + + # 创建测试数据 + time_range = pd.date_range('2020-01-01', '2020-01-10', freq='H') + basin_names = ['station_1', 'station_2', 'station_3', 'station_4', 'station_5', 'station_6'] + + # 生成随机数据 + np.random.seed(42) + precipitation_data = np.random.rand(len(basin_names), len(time_range)) * 10 + streamflow_data = np.random.rand(len(basin_names), len(time_range)) * 50 + + # 创建xarray Dataset + timeseries_dataset = xr.Dataset({ + 'total_precipitation_hourly': (['basin', 'time'], precipitation_data), + 'streamflow': (['basin', 'time'], streamflow_data) + }, coords={ + 'basin': basin_names, + 'time': time_range + }) + + timeseries_file = os.path.join(test_data_dir, "timeseries", "1D", "timeseries_1D.nc") + timeseries_dataset.to_netcdf(timeseries_file) + + print(f"创建文件: {timeseries_file}") + return timeseries_dataset + + +def create_test_shapes(test_data_dir): + """创建测试用的形状文件""" + print("创建测试形状文件...") + + basin_names = ['station_1', 'station_2', 'station_3', 'station_4', 'station_5', 'station_6'] + + # 创建简单的多边形几何 + geometries = [] + for i, basin_name in enumerate(basin_names): + # 创建简单的矩形多边形 (使用经纬度坐标) + x_offset = 100 + i * 0.1 # 经度 + y_offset = 30 + i * 0.1 # 纬度 + polygon = Polygon([ + (x_offset, y_offset), + (x_offset + 0.05, y_offset), + (x_offset + 0.05, y_offset + 0.05), + (x_offset, y_offset + 0.05) + ]) + geometries.append(polygon) + + # 创建GeoDataFrame并设置CRS为WGS84 + gdf = gpd.GeoDataFrame({ + 'BASIN_ID': basin_names, # 使用BASIN_ID列名 + 'geometry': geometries + }, crs='EPSG:4326') # WGS84坐标系 + + shapes_file = os.path.join(test_data_dir, "shapes", "basins.shp") + gdf.to_file(shapes_file) + + print(f"创建文件: {shapes_file}") + return gdf + + +def create_test_unit_info(test_data_dir): + """创建测试用的单位信息文件""" + print("创建测试单位信息文件...") + + # 创建单位信息字典 + unit_info = { + "ele_mt_sav": "m", + "area": "km2", + "p_mean": "mm", + "total_precipitation_hourly": "mm/h", + "streamflow": "mm/d" + } + + unit_file = os.path.join(test_data_dir, "timeseries", "1D_units_info.json") + with open(unit_file, 'w', encoding='utf-8') as f: + json.dump(unit_info, f, ensure_ascii=False, indent=2) + + print(f"创建文件: {unit_file}") + return unit_info + + +def create_test_attributes(test_data_dir): + """创建测试用的属性数据""" + print("创建测试属性数据...") + + basin_names = ['station_1', 'station_2', 'station_3', 'station_4', 'station_5', 'station_6'] + + # 生成随机属性数据 + np.random.seed(42) + ele_mt_sav = np.random.rand(len(basin_names)) * 1000 + 500 # 海拔 + area = np.random.rand(len(basin_names)) * 5000 + 1000 # 面积 + p_mean = np.random.rand(len(basin_names)) * 2000 + 800 # 平均降水 + + # 创建DataFrame并保存为CSV + attributes_df = pd.DataFrame({ + 'basin_id': basin_names, + 'ele_mt_sav': ele_mt_sav, + 'area': area, + 'p_mean': p_mean + }) + + attributes_file = os.path.join(test_data_dir, "attributes", "attributes.csv") + attributes_df.to_csv(attributes_file, index=False) + + print(f"创建文件: {attributes_file}") + return attributes_df + + +def test_tg_hydro_datasource(): + """测试TgHydroDatasource类""" + print("=" * 60) + print("开始测试TgHydroDatasource类") + print("=" * 60) + + # 设置测试数据目录 + test_data_dir = "/tmp/test_tg_hydro_data" + + try: + # 1. 创建测试数据 + create_test_data_structure(test_data_dir) + graph_dict = create_test_graph_dict(test_data_dir) + lstm_data = create_test_lstm_predictions(test_data_dir) + timeseries_data = create_test_timeseries(test_data_dir) + shapes_data = create_test_shapes(test_data_dir) + unit_info = create_test_unit_info(test_data_dir) + attributes_data = create_test_attributes(test_data_dir) + + # 2. 初始化TgHydroDatasource + print("\n" + "=" * 40) + print("测试TgHydroDatasource初始化") + print("=" * 40) + + tg_datasource = TgHydroDatasource( + data_path=test_data_dir, + time_unit=['1D'], + dataset_name="test_tg_dataset" + ) + + print(f"✓ TgHydroDatasource初始化成功") + print(f" 数据源名称: {tg_datasource.get_name()}") + print(f" 数据路径: {tg_datasource.data_source_dir}") + + # 3. 测试数据源描述 + print("\n" + "=" * 40) + print("测试数据源描述") + print("=" * 40) + + data_desc = tg_datasource.data_source_description + print("数据源描述包含的路径:") + for key, value in data_desc.items(): + if 'TGGNN' in key or 'LSTM' in key or 'GRAPH' in key: + print(f" {key}: {value}") + + # 4. 测试图网络结构加载 + print("\n" + "=" * 40) + print("测试图网络结构") + print("=" * 40) + + print(f"✓ 图网络结构加载成功") + print(f" 包含流域: {list(tg_datasource.graph_dict.keys())}") + + # 5. 测试LSTM预测数据读取 + print("\n" + "=" * 40) + print("测试LSTM预测数据读取") + print("=" * 40) + + # 获取实际可用的basin ID + available_basins = tg_datasource.read_object_ids() + test_basins = available_basins[:3] # 使用前三个basin + + lstm_predictions = tg_datasource.read_lstm_predictions( + object_ids=test_basins[:2], + t_range_list=['2020-01-01', '2020-01-05'] + ) + + # 5. 测试时间序列数据加载 + print("\n5. 测试时间序列数据加载...") + ts_data, lstm_data = tg_datasource.load_ts_data( + basin_names=test_basins[:2], + t_range=['2020-01-01', '2020-01-05'], + var_lst=['total_precipitation_hourly', 'streamflow'] + ) + print(f"✓ LSTM预测数据读取成功") + print(f" 数据形状: {lstm_predictions['streamflow'].shape}") + print(f" 时间范围: {lstm_predictions.time.values[0]} 到 {lstm_predictions.time.values[-1]}") + print(f"✓ 时间序列数据加载成功") + print(f" 时间序列数据形状: {ts_data.shape}") + print(f" LSTM数据形状: {lstm_data.shape}") + + # 6. 测试节点属性加载 + print("\n" + "=" * 40) + print("测试节点属性加载") + print("=" * 40) + node_attrs = tg_datasource.load_node_attributes( + basin_names=test_basins[:3], + selected_attrs=['ele_mt_sav', 'area', 'p_mean'] + ) + print(f"✓ 节点属性加载成功") + print(f" 属性张量形状: {node_attrs.shape}") + + # 7. 测试图生成 + print("\n" + "=" * 40) + print("测试图生成") + print("=" * 40) + + basin_names_all = ['station_1', 'station_2', 'station_3', 'station_4', 'station_5', 'station_6'] + dg, edges, node_mapping = tg_datasource.gen_nx_graph('test_basin', basin_names_all) + print(f"✓ 图生成成功") + print(f" 节点数量: {len(dg.nodes)}") + print(f" 边数量: {len(edges)}") + print(f" 节点映射: {node_mapping}") + + # 8. 测试上游节点查找 + print("\n" + "=" * 40) + print("测试上游节点查找") + print("=" * 40) + + upstream_nodes = tg_datasource.get_upstream_nodes('test_basin', 'station_3', basin_names_all) + print(f"✓ 上游节点查找成功") + print(f" station_3的上游节点索引: {upstream_nodes}") + + print("\n" + "=" * 60) + print("所有测试通过！TgHydroDatasource类工作正常") + print("=" * 60) + + return True + + except Exception as e: + print(f"\n❌ 测试失败: {str(e)}") + import traceback + traceback.print_exc() + return False + + finally: + # 清理测试数据 + import shutil + if os.path.exists(test_data_dir): + shutil.rmtree(test_data_dir) + print(f"\n清理测试数据目录: {test_data_dir}") + + +if __name__ == "__main__": + success = test_tg_hydro_datasource() + sys.exit(0 if success else 1) \ No newline at end of file From ca6e67de6581124d6c40e08aba4b6b5761187d45 Mon Sep 17 00:00:00 2001 From: owen Date: Sat, 22 Nov 2025 11:12:41 +0800 Subject: [PATCH 07/34] remove useless mdfiles --- ...02\346\225\260\350\257\264\346\230\216.md" | 176 ------------ ...45\345\217\243\350\257\264\346\230\216.md" | 225 --------------- ...02\346\225\260\350\257\264\346\230\216.md" | 195 ------------- .../example_tg_usage.py | 96 +++---- .../test_tg_hydro_datasource.py | 270 ++++++++++-------- 5 files changed, 197 insertions(+), 765 deletions(-) delete mode 100644 "TgHydroDatasource_\345\217\202\346\225\260\350\257\264\346\230\216.md" delete mode 100644 "TgHydroDatasource_\350\276\223\345\207\272\346\216\245\345\217\243\350\257\264\346\230\216.md" delete mode 100644 "read_lstm_predictions_\345\217\202\346\225\260\350\257\264\346\230\216.md" rename example_tg_usage.py => scripts/example_tg_usage.py (81%) rename test_tg_hydro_datasource.py => tests/test_tg_hydro_datasource.py (68%) diff --git "a/TgHydroDatasource_\345\217\202\346\225\260\350\257\264\346\230\216.md" "b/TgHydroDatasource_\345\217\202\346\225\260\350\257\264\346\230\216.md" deleted file mode 100644 index 8225755..0000000 --- "a/TgHydroDatasource_\345\217\202\346\225\260\350\257\264\346\230\216.md" +++ /dev/null @@ -1,176 +0,0 @@ -# TgHydroDatasource 参数说明文档 - -## 构造函数参数 - -### TgHydroDatasource 类的入参 - -```python -def __init__(self, data_path, download=False, time_unit=None, dataset_name=None, **kwargs): -``` - -### 参数详细说明 - -#### 1. 必需参数 - -| 参数名 | 类型 | 说明 | 示例 | -|--------|------|------|------| -| `data_path` | str | 数据根目录路径，可以是本地路径或S3路径 | `/path/to/data` 或 `s3://bucket/data` | - -#### 2. 可选参数 - -| 参数名 | 类型 | 默认值 | 说明 | 示例 | -|--------|------|--------|------|------| -| `download` | bool | False | 是否下载数据（如果数据源支持） | `True` 或 `False` | -| `time_unit` | list | ["1D"] | 时间单位列表，支持的单位有：["1h", "3h", "1D", "8D"] | `["1D"]` 或 `["1h", "3h"]` | -| `dataset_name` | str | None | 数据集名称，用于缓存文件命名等 | `"my_tg_dataset"` | -| `**kwargs` | dict | {} | 其他关键字参数，会传递给父类 | `{"version": "v1.0"}` | - -#### 3. kwargs 中支持的参数 - -| 参数名 | 类型 | 默认值 | 说明 | 示例 | -|--------|------|--------|------|------| -| `version` | str | None | 数据集版本号 | `"v2.0"` | - -## 参数传递机制 - -### 1. 继承关系 -``` -HydroData (基类) - ↓ -SelfMadeHydroDataset (父类) - ↓ -TgHydroDatasource (子类) -``` - -### 2. 参数传递流程 - -1. **TgHydroDatasource** 接收所有参数 -2. 通过 `super().__init__(data_path, download, time_unit, dataset_name, **kwargs)` 调用父类 -3. **SelfMadeHydroDataset** 处理自己的参数，再通过 `super().__init__(data_path)` 调用基类 -4. **HydroData** 基类只接收 `data_path` 参数 - -### 3. 参数处理详情 - -#### 在 SelfMadeHydroDataset 中： -```python -def __init__(self, data_path, download=False, time_unit=None, dataset_name=None, **kwargs): - # 参数验证和默认值设置 - if time_unit is None: - time_unit = ["1D"] - if any(unit not in ["1h", "3h", "1D", "8D"] for unit in time_unit): - raise ValueError("time_unit must be one of ['1h', '3h', '1D', '8D']") - - # 设置存储类型（本地或minio） - self.head = "minio" if "s3://" in data_path else "local" - - # 调用基类 - super().__init__(data_path) - - # 设置实例属性 - self.data_source_description = self.set_data_source_describe() - self.time_unit = time_unit - self.dataset_name = dataset_name - self.version = kwargs.get("version", None) # 从kwargs中提取version - - # 其他初始化 - if download: - self.download_data_source() - self.camels_sites = self.read_site_info() -``` - -#### 在 TgHydroDatasource 中： -```python -def __init__(self, data_path, download=False, time_unit=None, dataset_name=None, **kwargs): - # 调用父类初始化（所有参数都传递给父类） - super().__init__(data_path, download, time_unit, dataset_name, **kwargs) - - # TgHydroDatasource 特有的初始化 - self.graph_dict = self._load_graph_dict() -``` - -## 特殊参数处理 - -### 1. TgHydroDatasource 没有独有参数 -- TgHydroDatasource 的构造函数参数与父类 SelfMadeHydroDataset 完全相同 -- 所有参数都通过 `super().__init__()` 传递给父类处理 -- 子类只是在父类初始化完成后，添加了图网络结构的加载 - -### 2. 如果子类有独有参数的处理方式 - -假设 TgHydroDatasource 有独有参数，处理方式如下： - -```python -def __init__(self, data_path, download=False, time_unit=None, dataset_name=None, - graph_config=None, **kwargs): # graph_config 是子类独有参数 - """ - Parameters - ---------- - graph_config : dict, optional - 图网络配置参数（子类独有） - """ - # 处理子类独有参数 - self.graph_config = graph_config or {} - - # 调用父类初始化（不传递子类独有参数） - super().__init__(data_path, download, time_unit, dataset_name, **kwargs) - - # 子类特有的初始化 - self.graph_dict = self._load_graph_dict() -``` - -## 数据目录结构要求 - -TgHydroDatasource 期望的数据目录结构： - -``` -data_path/ -├── attributes/ # 流域属性数据（父类要求） -│ └── attributes.csv -├── timeseries/ # 时间序列数据（父类要求） -│ ├── 1D/ # 日数据 -│ ├── 3h/ # 3小时数据 -│ └── *_units_info.json -├── shapes/ # 流域形状文件（父类要求） -│ └── *.shp -└── tggnn/ # TG-GNN专用数据（子类要求） - ├── lstmpred.nc # LSTM预测数据 - └── graph_dict.json # 图网络结构 -``` - -## 使用示例 - -### 1. 基本使用 -```python -from hydrodatasource.reader.data_source import TgHydroDatasource - -# 最简单的使用方式 -tg_datasource = TgHydroDatasource(data_path="/path/to/data") -``` - -### 2. 完整参数使用 -```python -tg_datasource = TgHydroDatasource( - data_path="/path/to/data", - download=False, - time_unit=["1D", "3h"], - dataset_name="my_tg_dataset", - version="v1.0" -) -``` - -### 3. S3路径使用 -```python -tg_datasource = TgHydroDatasource( - data_path="s3://my-bucket/tg-data", - time_unit=["1D"], - dataset_name="s3_tg_dataset" -) -``` - -## 注意事项 - -1. **time_unit 限制**：只支持 ["1h", "3h", "1D", "8D"] 四种时间单位 -2. **数据路径**：支持本地路径和S3路径，S3路径需要以 "s3://" 开头 -3. **图网络文件**：必须存在 `tggnn/graph_dict.json` 文件，否则初始化会失败 -4. **继承关系**：所有父类的方法和属性都可以使用 -5. **参数传递**：kwargs 中的参数会传递给父类，可以用于扩展功能 \ No newline at end of file diff --git "a/TgHydroDatasource_\350\276\223\345\207\272\346\216\245\345\217\243\350\257\264\346\230\216.md" "b/TgHydroDatasource_\350\276\223\345\207\272\346\216\245\345\217\243\350\257\264\346\230\216.md" deleted file mode 100644 index 21fcd30..0000000 --- "a/TgHydroDatasource_\350\276\223\345\207\272\346\216\245\345\217\243\350\257\264\346\230\216.md" +++ /dev/null @@ -1,225 +0,0 @@ -# TgHydroDatasource 类输出接口说明 - -## 概述 - -`TgHydroDatasource` 类继承自 `SelfMadeHydroDataset`，提供了丰富的数据读取接口，主要用于图神经网络相关的水文建模任务。该类的输出接口可以分为以下几个类别： - -## 1. 基础数据读取接口（继承自父类） - -### 1.1 流域ID读取 -```python -def read_object_ids(self, object_params=None) -> np.array -``` -- **功能**: 读取所有可用的流域ID -- **返回**: numpy数组，包含所有流域ID -- **示例**: `basin_ids = tg_datasource.read_object_ids()` - -### 1.2 时间序列数据读取 -```python -def read_timeseries(self, object_ids=None, t_range_list=None, relevant_cols=None, **kwargs) -> dict -``` -- **功能**: 读取原始时间序列数据 -- **参数**: - - `object_ids`: 流域ID列表 - - `t_range_list`: 时间范围 `[start_time, end_time]` - - `relevant_cols`: 变量列表 - - `time_units`: 时间单位（如 `["1D", "3h"]`） -- **返回**: 字典，键为时间单位，值为numpy数组 `(basin_num, time_num, var_num)` - -### 1.3 属性数据读取 -```python -def read_attributes(self, object_ids=None, constant_cols=None, **kwargs) -> np.array -``` -- **功能**: 读取流域静态属性数据 -- **参数**: - - `object_ids`: 流域ID列表 - - `constant_cols`: 属性列名列表 -- **返回**: numpy数组 `(basin_num, attr_num)` - -### 1.4 xarray格式数据读取 -```python -def read_ts_xrdataset(self, gage_id_lst=None, t_range=None, var_lst=None, **kwargs) -> dict -``` -- **功能**: 读取xarray格式的时间序列数据 -- **返回**: 字典，键为时间单位，值为xarray.Dataset - -```python -def read_attr_xrdataset(self, gage_id_lst=None, var_lst=None, **kwargs) -> xr.Dataset -``` -- **功能**: 读取xarray格式的属性数据 -- **返回**: xarray.Dataset - -### 1.5 特定数据读取 -```python -def read_area(self, gage_id_lst=None) -> xr.Dataset -``` -- **功能**: 读取流域面积数据 -- **返回**: xarray.Dataset，包含面积信息 - -```python -def read_mean_prcp(self, gage_id_lst=None, unit="mm/d") -> xr.Dataset -``` -- **功能**: 读取平均降水数据 -- **返回**: xarray.Dataset，包含降水信息 - -## 2. TgHydroDatasource 特有接口 - -### 2.1 LSTM预测数据读取 -```python -def read_lstm_predictions(self, object_ids=None, t_range_list=None, **kwargs) -> xr.Dataset -``` -- **功能**: 读取LSTM预测数据 -- **参数**: - - `object_ids`: 流域ID列表 - - `t_range_list`: 时间范围 `[start_time, end_time]` -- **返回**: xarray.Dataset，包含LSTM预测结果 -- **数据来源**: `tggnn/lstmpred.nc` 文件 - -### 2.2 时间序列与LSTM数据联合读取 -```python -def read_timeseries_with_lstm(self, object_ids=None, t_range=None, var_lst=None, time_units=None) -> tuple -``` -- **功能**: 同时读取时间序列数据和LSTM预测数据 -- **参数**: - - `object_ids`: 流域ID列表 - - `t_range`: 时间范围 `[start_time, end_time]` - - `var_lst`: 变量列表（默认: `['total_precipitation_hourly', 'streamflow']`） - - `time_units`: 时间单位 -- **返回**: 元组 `(timeseries_data, lstm_data)`，两个都是xarray.Dataset - -### 2.3 节点属性数据读取 -```python -def read_node_attributes(self, object_ids=None, selected_attrs=None) -> xr.Dataset -``` -- **功能**: 读取节点静态属性数据（简化版，无PyTorch转换） -- **参数**: - - `object_ids`: 流域ID列表 - - `selected_attrs`: 属性列表（默认: `['ele_mt_sav', 'area', 'p_mean']`） -- **返回**: xarray.Dataset，包含筛选后的属性数据 - -### 2.4 图网络结构生成 -```python -def gen_nx_graph(self, basin_id, object_ids=None) -> tuple -``` -- **功能**: 生成NetworkX图对象，确保节点索引与数据一致 -- **参数**: - - `basin_id`: 流域ID（用于查找图结构） - - `object_ids`: 对象ID列表（可选） -- **返回**: 元组 `(NetworkX图对象, 边列表, 节点映射字典, 有效对象ID列表)` -- **数据来源**: `tggnn/graph_dict.json` 文件 - -### 2.5 上游节点查询 -```python -def get_upstream_nodes(self, basin_id, target_node, object_ids=None) -> list -``` -- **功能**: 获取指定节点的直接上游节点 -- **参数**: - - `basin_id`: 流域ID - - `target_node`: 目标节点名称 - - `object_ids`: 所有节点名称列表（可选） -- **返回**: 直接上游节点的索引列表 - -## 3. 元数据和信息接口 - -### 3.1 数据源信息 -```python -def get_name(self) -> str -``` -- **功能**: 返回数据源名称 -- **返回**: `"TgHydroDatasource"` - -```python -def set_data_source_describe(self) -> collections.OrderedDict -``` -- **功能**: 设置数据源描述，包含所有数据路径 -- **返回**: 有序字典，包含数据路径信息 - -### 3.2 列信息获取 -```python -def get_attributes_cols(self) -> np.array -``` -- **功能**: 获取属性数据的列名 -- **返回**: numpy数组，包含所有属性列名 - -```python -def get_timeseries_cols(self) -> dict -``` -- **功能**: 获取时间序列数据的列名 -- **返回**: 字典，键为时间单位，值为变量列表 - -## 4. 数据格式说明 - -### 4.1 返回数据类型 -- **numpy.array**: 原始数据读取方法返回 -- **xarray.Dataset**: 现代化数据读取方法返回，包含坐标、属性和元数据 -- **dict**: 多时间单位数据返回 -- **tuple**: 多种数据类型组合返回 - -### 4.2 坐标系统 -- **basin**: 流域维度，使用流域ID作为坐标 -- **time**: 时间维度，使用pandas时间索引 -- **变量维度**: 根据具体数据类型而定 - -### 4.3 数据筛选 -所有读取方法都支持： -- 按流域ID筛选（`object_ids`参数） -- 按时间范围筛选（`t_range`或`t_range_list`参数） -- 按变量筛选（`var_lst`或`selected_attrs`参数） - -## 5. 使用建议 - -### 5.1 推荐使用的接口 -- **现代化接口**: 优先使用返回xarray.Dataset的方法 -- **联合读取**: 使用`read_timeseries_with_lstm`同时获取多种数据 -- **图网络**: 使用`gen_nx_graph`获取完整的图结构信息 - -### 5.2 性能考虑 -- 大数据量时使用缓存机制（`cache_xrdataset`） -- 按需读取，避免加载不必要的数据 -- 合理设置时间范围和变量列表 - -### 5.3 错误处理 -- 所有方法都包含数据验证和错误提示 -- 缺失数据会给出警告信息 -- 支持部分数据缺失的情况下继续处理 - -## 6. 示例用法 - -```python -from hydrodatasource.reader.data_source import TgHydroDatasource - -# 初始化 -tg_datasource = TgHydroDatasource(data_path="/path/to/data") - -# 获取基础信息 -basin_ids = tg_datasource.read_object_ids() -print(f"可用流域数量: {len(basin_ids)}") - -# 读取时间序列和LSTM数据 -timeseries_data, lstm_data = tg_datasource.read_timeseries_with_lstm( - object_ids=basin_ids[:5], - t_range=['2020-01-01', '2020-12-31'], - var_lst=['streamflow', 'total_precipitation_hourly'] -) - -# 读取属性数据 -attr_data = tg_datasource.read_node_attributes( - object_ids=basin_ids[:5], - selected_attrs=['area', 'ele_mt_sav'] -) - -# 生成图网络 -graph, edges, node_mapping, valid_ids = tg_datasource.gen_nx_graph( - basin_id='sanxia_60703800', - object_ids=basin_ids[:10] -) - -# 查询上游节点 -upstream_nodes = tg_datasource.get_upstream_nodes( - basin_id='sanxia_60703800', - target_node='sanxia_60703780', - object_ids=valid_ids -) -``` - -这个接口设计提供了从基础数据读取到高级图网络分析的完整功能，特别适合图神经网络相关的水文建模任务。 \ No newline at end of file diff --git "a/read_lstm_predictions_\345\217\202\346\225\260\350\257\264\346\230\216.md" "b/read_lstm_predictions_\345\217\202\346\225\260\350\257\264\346\230\216.md" deleted file mode 100644 index 3215927..0000000 --- "a/read_lstm_predictions_\345\217\202\346\225\260\350\257\264\346\230\216.md" +++ /dev/null @@ -1,195 +0,0 @@ -# read_lstm_predictions 方法参数传递说明 - -## 方法签名 - -```python -def read_lstm_predictions(self, object_ids=None, t_range_list=None, **kwargs): -``` - -## 参数传递方式 - -### 1. 直接调用方式 - -用户可以直接调用`read_lstm_predictions`方法： - -```python -from hydrodatasource.reader.data_source import TgHydroDatasource - -# 初始化数据源 -tg_datasource = TgHydroDatasource(data_path="/path/to/data") - -# 直接调用read_lstm_predictions -lstm_data = tg_datasource.read_lstm_predictions( - object_ids=['basin_1', 'basin_2', 'basin_3'], - t_range_list=['2020-01-01', '2020-12-31'] -) -``` - -### 2. 通过read_timeseries_with_lstm间接调用 - -更常见的使用方式是通过`read_timeseries_with_lstm`方法间接调用： - -```python -# 通过read_timeseries_with_lstm调用 -timeseries_data, lstm_data = tg_datasource.read_timeseries_with_lstm( - object_ids=['basin_1', 'basin_2', 'basin_3'], # 这个参数会传递给read_lstm_predictions - t_range=['2020-01-01', '2020-12-31'], # 这个参数会传递给read_lstm_predictions的t_range_list - var_lst=['streamflow', 'precipitation'] -) -``` - -## 参数详细说明 - -### object_ids 参数 - -| 属性 | 说明 | -|------|------| -| **类型** | `list` 或 `None` | -| **默认值** | `None` | -| **作用** | 指定要读取的流域ID列表 | -| **传递方式** | 直接传递或通过`read_timeseries_with_lstm`的`object_ids`参数传递 | - -**使用示例：** -```python -# 方式1：直接传递 -lstm_data = tg_datasource.read_lstm_predictions( - object_ids=['basin_001', 'basin_002', 'basin_003'] -) - -# 方式2：通过read_timeseries_with_lstm传递 -timeseries_data, lstm_data = tg_datasource.read_timeseries_with_lstm( - object_ids=['basin_001', 'basin_002', 'basin_003'] # 会传递给read_lstm_predictions -) -``` - -**内部处理逻辑：** -```python -# 在read_lstm_predictions方法内部 -if object_ids is not None: - lstm_data = lstm_data.sel(basin=object_ids) # 使用xarray的sel方法筛选流域 -``` - -### t_range_list 参数 - -| 属性 | 说明 | -|------|------| -| **类型** | `list` 或 `None` | -| **默认值** | `None` | -| **格式** | `[start_time, end_time]` | -| **作用** | 指定要读取的时间范围 | -| **传递方式** | 直接传递或通过`read_timeseries_with_lstm`的`t_range`参数传递 | - -**使用示例：** -```python -# 方式1：直接传递 -lstm_data = tg_datasource.read_lstm_predictions( - t_range_list=['2020-01-01', '2020-12-31'] -) - -# 方式2：通过read_timeseries_with_lstm传递 -timeseries_data, lstm_data = tg_datasource.read_timeseries_with_lstm( - t_range=['2020-01-01', '2020-12-31'] # 会传递给read_lstm_predictions的t_range_list -) -``` - -**内部处理逻辑：** -```python -# 在read_lstm_predictions方法内部 -if t_range_list is not None: - start_time = pd.to_datetime(t_range_list[0]) - end_time = pd.to_datetime(t_range_list[1]) - lstm_data = lstm_data.sel(time=slice(start_time, end_time)) # 使用xarray的时间切片 -``` - -## 参数传递流程图 - -``` -用户调用 - ↓ -┌─────────────────────────────────────────────────────────────┐ -│ 方式1: 直接调用 │ -│ tg_datasource.read_lstm_predictions( │ -│ object_ids=['basin_1', 'basin_2'], │ -│ t_range_list=['2020-01-01', '2020-12-31'] │ -│ ) │ -└─────────────────────────────────────────────────────────────┘ - ↓ -read_lstm_predictions方法接收参数 - ↓ -处理和筛选数据 - -┌─────────────────────────────────────────────────────────────┐ -│ 方式2: 通过read_timeseries_with_lstm间接调用 │ -│ tg_datasource.read_timeseries_with_lstm( │ -│ object_ids=['basin_1', 'basin_2'], │ -│ t_range=['2020-01-01', '2020-12-31'], │ -│ var_lst=['streamflow'] │ -│ ) │ -└─────────────────────────────────────────────────────────────┘ - ↓ -read_timeseries_with_lstm方法内部调用 - ↓ -self.read_lstm_predictions( - object_ids=object_ids, # 直接传递 - t_range_list=t_range # 参数名映射：t_range → t_range_list -) - ↓ -read_lstm_predictions方法接收参数 - ↓ -处理和筛选数据 -``` - -## 参数验证和错误处理 - -### object_ids 验证 -- 如果传入的`object_ids`在数据集中不存在，xarray会抛出`KeyError` -- 建议先使用`tg_datasource.read_object_ids()`获取可用的流域ID - -### t_range_list 验证 -- 时间格式会通过`pd.to_datetime()`自动解析 -- 支持多种时间格式：'2020-01-01', '2020/01/01', '20200101'等 -- 如果时间范围超出数据集范围，会返回空的时间切片 - -## 完整使用示例 - -```python -from hydrodatasource.reader.data_source import TgHydroDatasource - -# 初始化 -tg_datasource = TgHydroDatasource(data_path="/path/to/data") - -# 获取可用的流域ID -available_basins = tg_datasource.read_object_ids() -print(f"可用流域: {available_basins[:5]}") - -# 选择要处理的流域和时间范围 -selected_basins = available_basins[:3] -time_range = ['2020-01-01', '2020-12-31'] - -# 方式1：直接调用read_lstm_predictions -lstm_data_direct = tg_datasource.read_lstm_predictions( - object_ids=selected_basins, - t_range_list=time_range -) - -# 方式2：通过read_timeseries_with_lstm调用 -timeseries_data, lstm_data_indirect = tg_datasource.read_timeseries_with_lstm( - object_ids=selected_basins, - t_range=time_range, - var_lst=['streamflow', 'precipitation'] -) - -# 两种方式得到的lstm_data应该是相同的 -print(f"直接调用结果: {lstm_data_direct.dims}") -print(f"间接调用结果: {lstm_data_indirect.dims}") -``` - -## 注意事项 - -1. **参数名映射**：`read_timeseries_with_lstm`中的`t_range`参数会映射为`read_lstm_predictions`中的`t_range_list`参数 - -2. **数据筛选顺序**：先按流域筛选，再按时间筛选 - -3. **性能考虑**：如果数据集很大，建议先筛选时间范围再筛选流域，以提高性能 - -4. **错误处理**：建议在调用前验证参数的有效性，避免运行时错误 \ No newline at end of file diff --git a/example_tg_usage.py b/scripts/example_tg_usage.py similarity index 81% rename from example_tg_usage.py rename to scripts/example_tg_usage.py index 623258f..8dbe39d 100644 --- a/example_tg_usage.py +++ b/scripts/example_tg_usage.py @@ -20,132 +20,122 @@ def example_usage(): """展示TgHydroDatasource的基本使用方法""" - + # 假设数据路径 data_path = "/path/to/your/tg_hydro_data" - + # 初始化TgHydroDatasource tg_datasource = TgHydroDatasource( - data_path=data_path, - time_unit=['1D'], - dataset_name="example_dataset" + data_path=data_path, time_unit=["1D"], dataset_name="example_dataset" ) - + print(f"数据源名称: {tg_datasource.get_name()}") print(f"数据路径: {tg_datasource.data_source_dir}") - + # 1. 读取可用的流域ID available_basins = tg_datasource.read_object_ids() print(f"可用流域数量: {len(available_basins)}") print(f"前5个流域ID: {available_basins[:5]}") - + # 2. 读取时间序列数据和LSTM预测数据 selected_basins = available_basins[:3] # 选择前3个流域 - time_range = ['2020-01-01', '2020-12-31'] - variables = ['total_precipitation_hourly', 'streamflow'] - + time_range = ["2020-01-01", "2020-12-31"] + variables = ["total_precipitation_hourly", "streamflow"] + timeseries_data, lstm_data = tg_datasource.read_timeseries_with_lstm( - object_ids=selected_basins, - t_range=time_range, - var_lst=variables + object_ids=selected_basins, t_range=time_range, var_lst=variables ) - + print(f"\n时间序列数据:") print(f" 数据类型: {type(timeseries_data)}") print(f" 时间单位: {list(timeseries_data.keys())}") for time_unit, data in timeseries_data.items(): print(f" {time_unit} 数据形状: {data.dims}") print(f" {time_unit} 变量: {list(data.data_vars)}") - + print(f"\nLSTM预测数据:") print(f" 数据类型: {type(lstm_data)}") print(f" 数据形状: {lstm_data.dims}") print(f" 变量: {list(lstm_data.data_vars)}") - + # 3. 读取属性数据 - selected_attrs = ['ele_mt_sav', 'area', 'p_mean'] + selected_attrs = ["ele_mt_sav", "area", "p_mean"] attr_data = tg_datasource.read_node_attributes( - object_ids=selected_basins, - selected_attrs=selected_attrs + object_ids=selected_basins, selected_attrs=selected_attrs ) - + print(f"\n属性数据:") print(f" 数据类型: {type(attr_data)}") print(f" 数据形状: {attr_data.dims}") print(f" 变量: {list(attr_data.data_vars)}") - + # 4. 生成图网络结构 basin_id = "test_basin" # 根据实际情况修改 try: dg, edges, node_mapping, valid_object_ids = tg_datasource.gen_nx_graph( - basin_id=basin_id, - object_ids=selected_basins + basin_id=basin_id, object_ids=selected_basins ) - + print(f"\n图网络结构:") print(f" 节点数量: {len(dg.nodes)}") print(f" 边数量: {len(edges)}") print(f" 节点映射: {node_mapping}") print(f" 有效对象ID: {valid_object_ids}") - + # 5. 获取上游节点 if valid_object_ids: target_node = valid_object_ids[0] upstream_nodes = tg_datasource.get_upstream_nodes( - basin_id=basin_id, - target_node=target_node, - object_ids=selected_basins + basin_id=basin_id, target_node=target_node, object_ids=selected_basins ) print(f" {target_node} 的上游节点索引: {upstream_nodes}") - + except ValueError as e: print(f"\n图网络结构错误: {e}") print("请检查graph_dict.json文件是否存在且包含正确的流域ID") - + # 6. 使用父类方法读取数据（展示继承功能） print(f"\n使用父类方法读取数据:") - + # 读取时间序列数据 ts_data = tg_datasource.read_ts_xrdataset( gage_id_lst=selected_basins, t_range=time_range, - var_lst=['streamflow'], - time_units=['1D'] + var_lst=["streamflow"], + time_units=["1D"], ) print(f" 时间序列数据 (父类方法): {type(ts_data)}") - + # 读取属性数据 - attr_data_parent = tg_datasource.read_attr_xrdataset( - var_lst=['area'] - ) + attr_data_parent = tg_datasource.read_attr_xrdataset(var_lst=["area"]) print(f" 属性数据 (父类方法): {type(attr_data_parent)}") def data_processing_example(): """展示如何处理读取的数据""" - - print("\n" + "="*50) + + print("\n" + "=" * 50) print("数据处理示例") - print("="*50) - + print("=" * 50) + # 这里展示如何处理从TgHydroDatasource读取的xarray数据 # 用户可以根据需要进行各种数据处理和分析 - + print("1. 数据筛选和切片") print(" - 使用 .sel() 方法按坐标筛选") print(" - 使用 .isel() 方法按索引筛选") print(" - 使用 .where() 方法按条件筛选") - + print("\n2. 数据转换") print(" - 使用 .values 获取numpy数组") print(" - 使用 .to_pandas() 转换为pandas") print(" - 使用 .to_netcdf() 保存为NetCDF文件") - + print("\n3. 数据分析") print(" - 使用 .mean(), .std(), .max(), .min() 等统计方法") print(" - 使用 .groupby() 进行分组分析") print(" - 使用 .resample() 进行时间重采样") - + print("\n4. 如果需要torch张量，可以手动转换:") print(" import torch") print(" tensor = torch.from_numpy(data.values).float()") @@ -153,16 +143,16 @@ def data_processing_example(): if __name__ == "__main__": print("TgHydroDatasource 简化版使用示例") - print("="*50) - + print("=" * 50) + try: example_usage() data_processing_example() - - print("\n" + "="*50) + + print("\n" + "=" * 50) print("示例完成！") - print("="*50) - + print("=" * 50) + except Exception as e: print(f"运行示例时出错: {e}") - print("请确保数据路径正确且数据文件存在") \ No newline at end of file + print("请确保数据路径正确且数据文件存在") diff --git a/test_tg_hydro_datasource.py b/tests/test_tg_hydro_datasource.py similarity index 68% rename from test_tg_hydro_datasource.py rename to tests/test_tg_hydro_datasource.py index 3e8563f..d2f2a09 100644 --- a/test_tg_hydro_datasource.py +++ b/tests/test_tg_hydro_datasource.py @@ -24,15 +24,10 @@ def create_test_data_structure(test_data_dir): """创建测试数据目录结构""" print("创建测试数据目录结构...") - + # 创建目录结构 - dirs_to_create = [ - "attributes", - "timeseries/1D", - "shapes", - "tggnn" - ] - + dirs_to_create = ["attributes", "timeseries/1D", "shapes", "tggnn"] + for dir_name in dirs_to_create: dir_path = os.path.join(test_data_dir, dir_name) os.makedirs(dir_path, exist_ok=True) @@ -42,19 +37,19 @@ def create_test_data_structure(test_data_dir): def create_test_graph_dict(test_data_dir): """创建测试用的graph_dict.json文件""" print("创建测试graph_dict.json文件...") - + graph_dict = { - 'test_basin': [ - ['station_1', 'station_2', 'station_3'], - ['station_4', 'station_2', 'station_3'], - ['station_5', 'station_6', 'station_3'] + "test_basin": [ + ["station_1", "station_2", "station_3"], + ["station_4", "station_2", "station_3"], + ["station_5", "station_6", "station_3"], ] } - + graph_dict_file = os.path.join(test_data_dir, "tggnn", "graph_dict.json") - with open(graph_dict_file, 'w', encoding='utf-8') as f: + with open(graph_dict_file, "w", encoding="utf-8") as f: json.dump(graph_dict, f, ensure_ascii=False, indent=2) - + print(f"创建文件: {graph_dict_file}") return graph_dict @@ -62,26 +57,31 @@ def create_test_graph_dict(test_data_dir): def create_test_lstm_predictions(test_data_dir): """创建测试用的lstmpred.nc文件""" print("创建测试lstmpred.nc文件...") - + # 创建测试数据 - time_range = pd.date_range('2020-01-01', '2020-01-10', freq='H') - basin_names = ['station_1', 'station_2', 'station_3', 'station_4', 'station_5', 'station_6'] - + time_range = pd.date_range("2020-01-01", "2020-01-10", freq="H") + basin_names = [ + "station_1", + "station_2", + "station_3", + "station_4", + "station_5", + "station_6", + ] + # 生成随机流量数据 np.random.seed(42) streamflow_data = np.random.rand(len(basin_names), len(time_range)) * 100 - + # 创建xarray Dataset - lstm_dataset = xr.Dataset({ - 'streamflow': (['basin', 'time'], streamflow_data) - }, coords={ - 'basin': basin_names, - 'time': time_range - }) - + lstm_dataset = xr.Dataset( + {"streamflow": (["basin", "time"], streamflow_data)}, + coords={"basin": basin_names, "time": time_range}, + ) + lstm_pred_file = os.path.join(test_data_dir, "tggnn", "lstmpred.nc") lstm_dataset.to_netcdf(lstm_pred_file) - + print(f"创建文件: {lstm_pred_file}") return lstm_dataset @@ -89,28 +89,37 @@ def create_test_lstm_predictions(test_data_dir): def create_test_timeseries(test_data_dir): """创建测试用的时间序列数据""" print("创建测试时间序列数据...") - + # 创建测试数据 - time_range = pd.date_range('2020-01-01', '2020-01-10', freq='H') - basin_names = ['station_1', 'station_2', 'station_3', 'station_4', 'station_5', 'station_6'] - + time_range = pd.date_range("2020-01-01", "2020-01-10", freq="H") + basin_names = [ + "station_1", + "station_2", + "station_3", + "station_4", + "station_5", + "station_6", + ] + # 生成随机数据 np.random.seed(42) precipitation_data = np.random.rand(len(basin_names), len(time_range)) * 10 streamflow_data = np.random.rand(len(basin_names), len(time_range)) * 50 - + # 创建xarray Dataset - timeseries_dataset = xr.Dataset({ - 'total_precipitation_hourly': (['basin', 'time'], precipitation_data), - 'streamflow': (['basin', 'time'], streamflow_data) - }, coords={ - 'basin': basin_names, - 'time': time_range - }) - - timeseries_file = os.path.join(test_data_dir, "timeseries", "1D", "timeseries_1D.nc") + timeseries_dataset = xr.Dataset( + { + "total_precipitation_hourly": (["basin", "time"], precipitation_data), + "streamflow": (["basin", "time"], streamflow_data), + }, + coords={"basin": basin_names, "time": time_range}, + ) + + timeseries_file = os.path.join( + test_data_dir, "timeseries", "1D", "timeseries_1D.nc" + ) timeseries_dataset.to_netcdf(timeseries_file) - + print(f"创建文件: {timeseries_file}") return timeseries_dataset @@ -118,32 +127,41 @@ def create_test_timeseries(test_data_dir): def create_test_shapes(test_data_dir): """创建测试用的形状文件""" print("创建测试形状文件...") - - basin_names = ['station_1', 'station_2', 'station_3', 'station_4', 'station_5', 'station_6'] - + + basin_names = [ + "station_1", + "station_2", + "station_3", + "station_4", + "station_5", + "station_6", + ] + # 创建简单的多边形几何 geometries = [] for i, basin_name in enumerate(basin_names): # 创建简单的矩形多边形 (使用经纬度坐标) x_offset = 100 + i * 0.1 # 经度 - y_offset = 30 + i * 0.1 # 纬度 - polygon = Polygon([ - (x_offset, y_offset), - (x_offset + 0.05, y_offset), - (x_offset + 0.05, y_offset + 0.05), - (x_offset, y_offset + 0.05) - ]) + y_offset = 30 + i * 0.1 # 纬度 + polygon = Polygon( + [ + (x_offset, y_offset), + (x_offset + 0.05, y_offset), + (x_offset + 0.05, y_offset + 0.05), + (x_offset, y_offset + 0.05), + ] + ) geometries.append(polygon) - + # 创建GeoDataFrame并设置CRS为WGS84 - gdf = gpd.GeoDataFrame({ - 'BASIN_ID': basin_names, # 使用BASIN_ID列名 - 'geometry': geometries - }, crs='EPSG:4326') # WGS84坐标系 - + gdf = gpd.GeoDataFrame( + {"BASIN_ID": basin_names, "geometry": geometries}, # 使用BASIN_ID列名 + crs="EPSG:4326", + ) # WGS84坐标系 + shapes_file = os.path.join(test_data_dir, "shapes", "basins.shp") gdf.to_file(shapes_file) - + print(f"创建文件: {shapes_file}") return gdf @@ -151,20 +169,20 @@ def create_test_shapes(test_data_dir): def create_test_unit_info(test_data_dir): """创建测试用的单位信息文件""" print("创建测试单位信息文件...") - + # 创建单位信息字典 unit_info = { "ele_mt_sav": "m", - "area": "km2", + "area": "km2", "p_mean": "mm", "total_precipitation_hourly": "mm/h", - "streamflow": "mm/d" + "streamflow": "mm/d", } - + unit_file = os.path.join(test_data_dir, "timeseries", "1D_units_info.json") - with open(unit_file, 'w', encoding='utf-8') as f: + with open(unit_file, "w", encoding="utf-8") as f: json.dump(unit_info, f, ensure_ascii=False, indent=2) - + print(f"创建文件: {unit_file}") return unit_info @@ -172,26 +190,35 @@ def create_test_unit_info(test_data_dir): def create_test_attributes(test_data_dir): """创建测试用的属性数据""" print("创建测试属性数据...") - - basin_names = ['station_1', 'station_2', 'station_3', 'station_4', 'station_5', 'station_6'] - + + basin_names = [ + "station_1", + "station_2", + "station_3", + "station_4", + "station_5", + "station_6", + ] + # 生成随机属性数据 np.random.seed(42) ele_mt_sav = np.random.rand(len(basin_names)) * 1000 + 500 # 海拔 - area = np.random.rand(len(basin_names)) * 5000 + 1000 # 面积 - p_mean = np.random.rand(len(basin_names)) * 2000 + 800 # 平均降水 - + area = np.random.rand(len(basin_names)) * 5000 + 1000 # 面积 + p_mean = np.random.rand(len(basin_names)) * 2000 + 800 # 平均降水 + # 创建DataFrame并保存为CSV - attributes_df = pd.DataFrame({ - 'basin_id': basin_names, - 'ele_mt_sav': ele_mt_sav, - 'area': area, - 'p_mean': p_mean - }) - + attributes_df = pd.DataFrame( + { + "basin_id": basin_names, + "ele_mt_sav": ele_mt_sav, + "area": area, + "p_mean": p_mean, + } + ) + attributes_file = os.path.join(test_data_dir, "attributes", "attributes.csv") attributes_df.to_csv(attributes_file, index=False) - + print(f"创建文件: {attributes_file}") return attributes_df @@ -201,10 +228,10 @@ def test_tg_hydro_datasource(): print("=" * 60) print("开始测试TgHydroDatasource类") print("=" * 60) - + # 设置测试数据目录 test_data_dir = "/tmp/test_tg_hydro_data" - + try: # 1. 创建测试数据 create_test_data_structure(test_data_dir) @@ -214,41 +241,39 @@ def test_tg_hydro_datasource(): shapes_data = create_test_shapes(test_data_dir) unit_info = create_test_unit_info(test_data_dir) attributes_data = create_test_attributes(test_data_dir) - + # 2. 初始化TgHydroDatasource print("\n" + "=" * 40) print("测试TgHydroDatasource初始化") print("=" * 40) - + tg_datasource = TgHydroDatasource( - data_path=test_data_dir, - time_unit=['1D'], - dataset_name="test_tg_dataset" + data_path=test_data_dir, time_unit=["1D"], dataset_name="test_tg_dataset" ) - + print(f"✓ TgHydroDatasource初始化成功") print(f" 数据源名称: {tg_datasource.get_name()}") print(f" 数据路径: {tg_datasource.data_source_dir}") - + # 3. 测试数据源描述 print("\n" + "=" * 40) print("测试数据源描述") print("=" * 40) - + data_desc = tg_datasource.data_source_description print("数据源描述包含的路径:") for key, value in data_desc.items(): - if 'TGGNN' in key or 'LSTM' in key or 'GRAPH' in key: + if "TGGNN" in key or "LSTM" in key or "GRAPH" in key: print(f" {key}: {value}") - + # 4. 测试图网络结构加载 print("\n" + "=" * 40) print("测试图网络结构") print("=" * 40) - + print(f"✓ 图网络结构加载成功") print(f" 包含流域: {list(tg_datasource.graph_dict.keys())}") - + # 5. 测试LSTM预测数据读取 print("\n" + "=" * 40) print("测试LSTM预测数据读取") @@ -259,71 +284,84 @@ def test_tg_hydro_datasource(): test_basins = available_basins[:3] # 使用前三个basin lstm_predictions = tg_datasource.read_lstm_predictions( - object_ids=test_basins[:2], - t_range_list=['2020-01-01', '2020-01-05'] + object_ids=test_basins[:2], t_range_list=["2020-01-01", "2020-01-05"] ) - + # 5. 测试时间序列数据加载 print("\n5. 测试时间序列数据加载...") ts_data, lstm_data = tg_datasource.load_ts_data( basin_names=test_basins[:2], - t_range=['2020-01-01', '2020-01-05'], - var_lst=['total_precipitation_hourly', 'streamflow'] + t_range=["2020-01-01", "2020-01-05"], + var_lst=["total_precipitation_hourly", "streamflow"], ) print(f"✓ LSTM预测数据读取成功") print(f" 数据形状: {lstm_predictions['streamflow'].shape}") - print(f" 时间范围: {lstm_predictions.time.values[0]} 到 {lstm_predictions.time.values[-1]}") + print( + f" 时间范围: {lstm_predictions.time.values[0]} 到 {lstm_predictions.time.values[-1]}" + ) print(f"✓ 时间序列数据加载成功") print(f" 时间序列数据形状: {ts_data.shape}") print(f" LSTM数据形状: {lstm_data.shape}") - + # 6. 测试节点属性加载 print("\n" + "=" * 40) print("测试节点属性加载") print("=" * 40) node_attrs = tg_datasource.load_node_attributes( - basin_names=test_basins[:3], - selected_attrs=['ele_mt_sav', 'area', 'p_mean'] + basin_names=test_basins[:3], selected_attrs=["ele_mt_sav", "area", "p_mean"] ) print(f"✓ 节点属性加载成功") print(f" 属性张量形状: {node_attrs.shape}") - + # 7. 测试图生成 print("\n" + "=" * 40) print("测试图生成") print("=" * 40) - - basin_names_all = ['station_1', 'station_2', 'station_3', 'station_4', 'station_5', 'station_6'] - dg, edges, node_mapping = tg_datasource.gen_nx_graph('test_basin', basin_names_all) + + basin_names_all = [ + "station_1", + "station_2", + "station_3", + "station_4", + "station_5", + "station_6", + ] + dg, edges, node_mapping = tg_datasource.gen_nx_graph( + "test_basin", basin_names_all + ) print(f"✓ 图生成成功") print(f" 节点数量: {len(dg.nodes)}") print(f" 边数量: {len(edges)}") print(f" 节点映射: {node_mapping}") - + # 8. 测试上游节点查找 print("\n" + "=" * 40) print("测试上游节点查找") print("=" * 40) - - upstream_nodes = tg_datasource.get_upstream_nodes('test_basin', 'station_3', basin_names_all) + + upstream_nodes = tg_datasource.get_upstream_nodes( + "test_basin", "station_3", basin_names_all + ) print(f"✓ 上游节点查找成功") print(f" station_3的上游节点索引: {upstream_nodes}") - + print("\n" + "=" * 60) print("所有测试通过！TgHydroDatasource类工作正常") print("=" * 60) - + return True - + except Exception as e: print(f"\n❌ 测试失败: {str(e)}") import traceback + traceback.print_exc() return False - + finally: # 清理测试数据 import shutil + if os.path.exists(test_data_dir): shutil.rmtree(test_data_dir) print(f"\n清理测试数据目录: {test_data_dir}") @@ -331,4 +369,4 @@ def test_tg_hydro_datasource(): if __name__ == "__main__": success = test_tg_hydro_datasource() - sys.exit(0 if success else 1) \ No newline at end of file + sys.exit(0 if success else 1) From aaf54a1a4dd9994f2936c1c5b0a877e9d4801814 Mon Sep 17 00:00:00 2001 From: owen Date: Sat, 22 Nov 2025 17:28:24 +0800 Subject: [PATCH 08/34] update tghydrodatasource init --- hydrodatasource/reader/data_source.py | 39 +++++++++++++++++++++------ 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/hydrodatasource/reader/data_source.py b/hydrodatasource/reader/data_source.py index b9853dd..e99c50d 100644 --- a/hydrodatasource/reader/data_source.py +++ b/hydrodatasource/reader/data_source.py @@ -1842,9 +1842,7 @@ class TgHydroDatasource(SelfMadeHydroDataset): └── graph_dict.json # 图网络结构 """ - def __init__( - self, data_path, download=False, time_unit=None, dataset_name=None, **kwargs - ): + def __init__(self, data_path, dataset_name=None, time_unit=None, **kwargs): """ 初始化TG流域数据集 @@ -1852,17 +1850,21 @@ def __init__( ---------- data_path : str 数据根目录路径 - download : bool, optional - 是否下载数据, by default False - time_unit : list, optional - 时间单位列表, by default None dataset_name : str, optional 数据集名称, by default None + time_unit : list, optional + 时间单位列表, by default None **kwargs : dict 其他参数 """ # 调用父类初始化 - super().__init__(data_path, download, time_unit, dataset_name, **kwargs) + self.inter_basin_pred_file = kwargs.get("inter_basin_pred_file", None) + if self.inter_basin_pred_file is None: + raise ValueError( + "inter_basin_pred_file is required; please run scripts/generate_inter_basin_predictions.py to generate the inter-basin predictions." + ) + kwargs.pop("inter_basin_pred_file") + super().__init__(data_path, dataset_name, time_unit, **kwargs) # 加载图网络结构 self.graph_dict = self._load_graph_dict() @@ -2287,3 +2289,24 @@ def classify_nodes(dg, basin_names): print(f"汇流节点 ({len(confluence_nodes)}个): {confluence_nodes}") return source_nodes, confluence_nodes, node_mask + + def read_ts_xrdataset( + self, + gage_id_lst: list = None, + t_range: list = None, + var_lst: list = None, + **kwargs, + ) -> dict: + """ + Read the time series data from the data source. + + TODO: Implement this function. + """ + pass + + def read_attr_xrdataset(self, gage_id_lst=None, var_lst=None, **kwargs): + """Read the attribute data from the data source. + + TODO: Implement this function. + """ + pass From 088c5d1115bdddd162cf5504d95edca2719671b8 Mon Sep 17 00:00:00 2001 From: mahaoran2022 <965297063@qq.com> Date: Fri, 12 Dec 2025 18:46:18 +0800 Subject: [PATCH 09/34] updata tggnn version 1.0 --- hydrodatasource/reader/data_source.py | 1112 +++++++++++++++++-------- tests/test_tg_hydro_datasource.py | 548 ++++++------ 2 files changed, 1045 insertions(+), 615 deletions(-) diff --git a/hydrodatasource/reader/data_source.py b/hydrodatasource/reader/data_source.py index e99c50d..46b8368 100644 --- a/hydrodatasource/reader/data_source.py +++ b/hydrodatasource/reader/data_source.py @@ -1832,14 +1832,16 @@ class TgHydroDatasource(SelfMadeHydroDataset): 该类在标准的水文数据集基础上，增加了对LSTM预测数据和图网络结构的支持， 主要用于图神经网络相关的水文建模任务。 + catch中应包含LSTM预测结果NC文件，文件名格式为lstmpred.nc 数据目录结构: - - attributes/ # 流域属性数据 - - timeseries/ # 时间序列数据 - - shapes/ # 流域形状文件 - - tggnn/ # TG-GNN专用数据 - ├── lstmpred.nc # LSTM预测数据 - └── graph_dict.json # 图网络结构 + - attributes/ # 控制流域属性数据 + - timeseries/ # 控制流域时间序列数据 + - shapes/ # 控制流域形状文件 + - intermediate/ # 区间流域数据 + ├── attributes/ # 区间流域属性数据，包含拓扑关系 + ├── timeseries/ # 区间流域时间序列数据 + └── shapes/ # 区间流域形状文件 """ def __init__(self, data_path, dataset_name=None, time_unit=None, **kwargs): @@ -1857,17 +1859,17 @@ def __init__(self, data_path, dataset_name=None, time_unit=None, **kwargs): **kwargs : dict 其他参数 """ - # 调用父类初始化 - self.inter_basin_pred_file = kwargs.get("inter_basin_pred_file", None) - if self.inter_basin_pred_file is None: - raise ValueError( - "inter_basin_pred_file is required; please run scripts/generate_inter_basin_predictions.py to generate the inter-basin predictions." - ) - kwargs.pop("inter_basin_pred_file") + # # 调用父类初始化 + # self.inter_basin_pred_file = kwargs.get("inter_basin_pred_file", None) + # if self.inter_basin_pred_file is None: + # raise ValueError( + # "inter_basin_pred_file is required; please run scripts/generate_inter_basin_predictions.py to generate the inter-basin predictions." + # ) + # kwargs.pop("inter_basin_pred_file") super().__init__(data_path, dataset_name, time_unit, **kwargs) - # 加载图网络结构 - self.graph_dict = self._load_graph_dict() + # 基于 intermediate 属性数据的 downstream_id 生成拓扑关系 + self.dg = self._build_graph_from_intermediate_attributes() def get_name(self): """返回数据源名称""" @@ -1883,412 +1885,522 @@ def set_data_source_describe(self): 包含所有数据路径的有序字典 """ # 获取父类的数据源描述 - data_source_desc = super().set_data_source_describe() + base_desc = super().set_data_source_describe() - # 添加TGGNN相关路径 + # ==== LSTM paths ==== data_root_dir = self.data_source_dir - tggnn_dir = os.path.join(data_root_dir, "tggnn") - lstm_pred_file = os.path.join(tggnn_dir, "lstmpred.nc") - graph_dict_file = os.path.join(tggnn_dir, "graph_dict.json") - # 更新数据源描述 + lstm_pred_file = os.path.join(CACHE_DIR, "lstmpred.nc") + + + + # ==== Intermediate dataset paths ==== + intermediate_root = os.path.join(data_root_dir, "intermediate") + inter_attr_file = os.path.join(intermediate_root, "attributes", "attributes.csv") + inter_shapes_dir = os.path.join(intermediate_root, "shapes") + inter_shape_file = os.path.join(inter_shapes_dir, "basins.shp") + inter_basinoutlets_file = os.path.join(inter_shapes_dir, "basinoutlets.shp") + inter_ts_dir = os.path.join(intermediate_root, "timeseries") + + # 列出 intermediate 下的时间单位目录（若存在） + inter_ts_dirs = [] + if ("s3://" in inter_ts_dir and is_minio_folder(inter_ts_dir)) or ( + "s3://" not in inter_ts_dir and os.path.isdir(inter_ts_dir) + ): + inter_ts_dirs = self._where_ts_dir(inter_ts_dir) + + # 组装并返回完整描述 + data_source_desc = collections.OrderedDict() + data_source_desc.update(base_desc) data_source_desc.update( { - "TGGNN_DIR": tggnn_dir, + # LSTM paths "LSTM_PRED_FILE": lstm_pred_file, - "GRAPH_DICT_FILE": graph_dict_file, + + # Intermediate paths + "INTERMEDIATE_ROOT_DIR": intermediate_root, + "INTERMEDIATE_ATTR_FILE": inter_attr_file, + "INTERMEDIATE_SHAPE_FILE": inter_shape_file, + "INTERMEDIATE_BASINOUTLETS_FILE": inter_basinoutlets_file, + "INTERMEDIATE_TS_DIRS": inter_ts_dirs, } ) return data_source_desc - def _load_graph_dict(self): + def _build_graph_from_intermediate_attributes(self): """ - 从JSON文件加载图网络结构 + 从 intermediate/attributes/attributes.csv 读取 downstream_id 列，生成有向拓扑图。 + + 约定： + - 使用列 `basin_id` 作为节点标识； + - 使用列 `downstream_id` 指定下游站点ID；当值为 0/空/NaN 表示为出口站（不连接下游）。 + - 仅在下游ID也存在于数据集中时建立边（忽略数据集外的下游指向）。 Returns ------- - dict - 图网络结构字典 + networkx.DiGraph + 由属性文件构建的有向图 """ - graph_dict_file = self.data_source_description["GRAPH_DICT_FILE"] + attr_file_inter = self._get_intermediate_attr_file() + # 若 intermediate 属性文件不存在，则优雅回退到基础属性文件 + if "s3://" in attr_file_inter: + try: + exists_inter = conf.FS.exists(attr_file_inter) + except Exception: + exists_inter = False + else: + exists_inter = os.path.exists(attr_file_inter) + attr_file = ( + attr_file_inter + if exists_inter + else self.data_source_description.get("ATTR_FILE", attr_file_inter) + ) + + # 读取 CSV（兼容本地与 s3）；不存在则返回空图 try: - if "s3://" in graph_dict_file: - with conf.FS.open(graph_dict_file, mode="rb") as f: - graph_dict = json.load(f) + if "s3://" in attr_file: + with conf.FS.open(attr_file, mode="rb") as f: + df = pd.read_csv(f, dtype={"basin_id": str}) else: - with open(graph_dict_file, "r", encoding="utf-8") as f: - graph_dict = json.load(f) + df = pd.read_csv(attr_file, dtype={"basin_id": str}) except FileNotFoundError: - # 如果文件不存在，返回默认的图网络结构 - print(f"警告：图网络结构文件 {graph_dict_file} 不存在，使用默认结构") - graph_dict = self._get_default_graph_dict() - # 保存默认结构到文件 - self._save_graph_dict(graph_dict) + print( + f"警告：未找到属性文件 {attr_file}；返回空图。请提供 intermediate/attributes/attributes.csv 或基础 attributes.csv" + ) + return nx.DiGraph() + + # 列校验 + if "basin_id" not in df.columns: + raise ValueError("attributes.csv 缺少必需列 'basin_id'") + if "downstream_id" not in df.columns: + # 若缺失下游列，则生成无边图，仅包含节点 + print("警告：attributes.csv 中未找到 'downstream_id' 列，生成无边图") + dg = nx.DiGraph() + dg.add_nodes_from(df["basin_id"].astype(str).tolist()) + return dg + + # 规范化下游列为字符串，并处理出口标识 + def _norm_downstream(val): + if pd.isna(val): + return None + s = str(val).strip() + if s in ("", "0", "0.0", "None", "nan", "NaN"): + return None + return s + + df["downstream_id"] = df["downstream_id"].apply(_norm_downstream) + + basin_set = set(df["basin_id"].astype(str).tolist()) + dg = nx.DiGraph() + dg.add_nodes_from(basin_set) + + # 建立边（仅当下游也在数据集内） + for _, row in df.iterrows(): + u = str(row["basin_id"]).strip() + v = row["downstream_id"] + if v is None: + continue + v = str(v).strip() + if v in basin_set: + dg.add_edge(u, v) - return graph_dict + return dg - def _get_default_graph_dict(self): + def _build_graph_and_index(self, object_ids=None): """ - 获取默认的图网络结构 + 合并构图与索引映射生成： + + - 基于 intermediate 属性构建全局有向图并赋值到 `self.dg`； + - 依据 `object_ids`（若提供）生成节点索引映射与边索引列表； + - 若未提供 `object_ids`，将使用数据集中所有可用对象ID与图节点的交集。 Returns ------- - dict - 默认图网络结构 + tuple + (NetworkX图对象, 边列表(按索引), 节点映射字典, 有效对象ID列表) """ - return { - "sanxia_60703800": [ - ["sanxia_60718300", "sanxia_60703780", "sanxia_60703800"], - [ - "sanxia_60716700", - "sanxia_60715400", - "sanxia_60715955", - "sanxia_60703780", - "sanxia_60703800", - ], - [ - "sanxia_60716250", - "sanxia_60715000", - "sanxia_60715400", - "sanxia_60715955", - "sanxia_60703780", - "sanxia_60703800", - ], - [ - "sanxia_60701001", - "sanxia_60715000", - "sanxia_60715400", - "sanxia_60715955", - "sanxia_60703780", - "sanxia_60703800", - ], - [ - "sanxia_60717050", - "sanxia_60715400", - "sanxia_60715955", - "sanxia_60703780", - "sanxia_60703800", - ], - [ - "sanxia_60710650", - "sanxia_60703700", - "sanxia_60703780", - "sanxia_60703800", - ], - [ - "sanxia_60706825", - "sanxia_60700001", - "sanxia_60700002", - "sanxia_60703700", - "sanxia_60703780", - "sanxia_60703800", - ], - [ - "sanxia_60701750", - "sanxia_60700002", - "sanxia_60703700", - "sanxia_60703780", - "sanxia_60703800", - ], - [ - "sanxia_60709859", - "sanxia_60703700", - "sanxia_60703780", - "sanxia_60703800", - ], - [ - "sanxia_60713800", - "sanxia_60712000", - "sanxia_60703780", - "sanxia_60703800", - ], - [ - "sanxia_60713730", - "sanxia_60711800", - "sanxia_60712000", - "sanxia_60703780", - "sanxia_60703800", - ], - [ - "sanxia_60713630", - "sanxia_60711800", - "sanxia_60712000", - "sanxia_60703780", - "sanxia_60703800", - ], - [ - "sanxia_60713400", - "sanxia_60706011", - "sanxia_60713170", - "sanxia_60711800", - "sanxia_60712000", - "sanxia_60703780", - "sanxia_60703800", - ], - [ - "sanxia_60713270", - "sanxia_60713170", - "sanxia_60711800", - "sanxia_60712000", - "sanxia_60703780", - "sanxia_60703800", - ], - [ - "sanxia_60712900", - "sanxia_60711800", - "sanxia_60712000", - "sanxia_60703780", - "sanxia_60703800", - ], - [ - "sanxia_60701101", - "sanxia_60711800", - "sanxia_60712000", - "sanxia_60703780", - "sanxia_60703800", - ], - [ - "sanxia_60711600", - "sanxia_60711800", - "sanxia_60712000", - "sanxia_60703780", - "sanxia_60703800", - ], - ] - } + # 确保全局图存在 + base_graph = getattr(self, "dg", None) + if base_graph is None or not isinstance(base_graph, nx.DiGraph): + base_graph = self._build_graph_from_intermediate_attributes() + self.dg = base_graph - def _save_graph_dict(self, graph_dict): - """ - 保存图网络结构到JSON文件 + graph_nodes = set(base_graph.nodes) - Parameters - ---------- - graph_dict : dict - 图网络结构字典 - """ - graph_dict_file = self.data_source_description["GRAPH_DICT_FILE"] + # 选择对象ID集合 + if object_ids is None: + available_basins = self.read_object_ids() + # 统一为字符串以避免类型不一致导致的交集为空 + valid_object_ids = [str(bid) for bid in available_basins if str(bid) in graph_nodes] + else: + object_ids_list = [str(bid) for bid in list(object_ids)] + valid_object_ids = [bid for bid in object_ids_list if bid in graph_nodes] + missing_in_graph = set(object_ids_list) - graph_nodes + if missing_in_graph: + print(f"警告：以下流域ID在图网络结构中不存在: {missing_in_graph}") - # 确保目录存在 - os.makedirs(os.path.dirname(graph_dict_file), exist_ok=True) + # 节点到索引的映射 + node_mapping = {node_id: idx for idx, node_id in enumerate(valid_object_ids)} - if "s3://" in graph_dict_file: - with conf.FS.open(graph_dict_file, mode="w") as f: - json.dump(graph_dict, f, ensure_ascii=False, indent=2) - else: - with open(graph_dict_file, "w", encoding="utf-8") as f: - json.dump(graph_dict, f, ensure_ascii=False, indent=2) + # 在全局图上生成边索引（仅保留筛选集合内的边） + edges = [] + for u, v in base_graph.edges: + if u in node_mapping and v in node_mapping: + edges.append((node_mapping[u], node_mapping[v])) + + return base_graph, edges, node_mapping, valid_object_ids + - def read_lstm_predictions(self, object_ids=None, t_range_list=None, **kwargs): + def read_graph_data(self, object_ids=None, as_numpy=False, include_optional=True): """ - 读取LSTM预测数据 + 统一返回下游所需的图拓扑结构，满足最大兼容性约定。 - Parameters - ---------- + 参数 + ---- object_ids : list, optional - 流域ID列表, by default None - t_range_list : list, optional - 时间范围列表, by default None - **kwargs : dict - 其他参数 - - Returns - ------- - xr.Dataset - LSTM预测数据集 + 指定参与图构建的节点集合；不传则使用数据集可用节点与图节点的交集 + as_numpy : bool, default False + 将 edge_index 返回为 numpy.int64 的 ndarray；默认返回 torch.long 的 Tensor + include_optional : bool, default True + 是否包含可选扩展字段（目前支持 upstream_indices） + + 返回 + ---- + dict + { + 'edge_index': [2, E] 的 torch.LongTensor 或 numpy.ndarray, + 'node_ids': List[str]，长度为 num_nodes，顺序与索引一致, + 'node_mask': List[bool]，长度为 num_nodes（汇流 True / 源头 False）, + 可选：'upstream_indices': List[List[int]] + } """ - lstm_pred_file = self.data_source_description["LSTM_PRED_FILE"] + # 获取基本图数据 + base_graph, edges, node_mapping, node_ids = self._build_graph_and_index( + object_ids=object_ids + ) - if "s3://" in lstm_pred_file: - with conf.FS.open(lstm_pred_file, mode="rb") as f: - lstm_data = xr.open_dataset(f) + # edge_index 构造 + import numpy as np + if len(edges) == 0: + if as_numpy: + edge_index = np.empty((2, 0), dtype=np.int64) + else: + try: + import torch + edge_index = torch.empty((2, 0), dtype=torch.long) + except Exception as e: + raise ImportError("read_graph_data 需要安装 PyTorch 或设置 as_numpy=True") from e else: - lstm_data = xr.open_dataset(lstm_pred_file) - - # 根据参数筛选数据 - if object_ids is not None: - lstm_data = lstm_data.sel(basin=object_ids) + e = np.array(edges, dtype=np.int64).T # [2, E] + if as_numpy: + edge_index = e + else: + try: + import torch + edge_index = torch.from_numpy(e).long() + except Exception as e: + raise ImportError("read_graph_data 需要安装 PyTorch 或设置 as_numpy=True") from e + + # 节点掩码（与 node_ids 顺序一致） + _, _, node_mask = self.classify_nodes(basin_names=node_ids) + + result = { + "edge_index": edge_index, + "node_ids": list(node_ids), + "node_mask": list(node_mask), + } - if t_range_list is not None: - start_time = pd.to_datetime(t_range_list[0]) - end_time = pd.to_datetime(t_range_list[1]) - lstm_data = lstm_data.sel(time=slice(start_time, end_time)) + # 可选扩展：上游节点索引列表 + if include_optional: + upstream_indices = [] + # node_ids 与 node_mapping 对齐为索引域 [0..N-1] + for nid in node_ids: + preds = list(base_graph.predecessors(nid)) + upstream_indices.append(sorted([node_mapping[p] for p in preds if p in node_mapping])) + result["upstream_indices"] = upstream_indices - return lstm_data + return result - def read_timeseries_with_lstm( - self, object_ids=None, t_range=None, var_lst=None, time_units=None - ): + def classify_nodes(self, basin_names=None): """ - 读取时间序列数据和LSTM预测数据 + 分类节点为源头节点和汇流节点。 - Parameters - ---------- - object_ids : list, optional - 对象ID列表（流域名称）, by default None - t_range : list, optional - 时间范围，格式为[start_time, end_time], by default None - var_lst : list, optional - 变量列表, by default None - time_units : list, optional - 时间单位，默认使用类的time_unit属性, by default None + - 默认在全局图 `self.dg` 上运行； + - 若 `basin_names` 为 None，则使用图中的所有节点，按数据集顺序（与 `read_object_ids` 对齐）。 Returns ------- tuple - (时间序列数据xarray.Dataset, LSTM预测数据xarray.Dataset) + (source_nodes, confluence_nodes, node_mask) """ - if time_units is None: - time_units = self.time_unit + dg = getattr(self, "dg", None) + if dg is None: + dg = self._build_graph_from_intermediate_attributes() + + if basin_names is None: + available_basins = self.read_object_ids() + basin_names = [bid for bid in available_basins if bid in dg.nodes] - var_lst_tggnn = ["total_precipitation_hourly", "discharge", "streamflow"] + source_nodes = [] + confluence_nodes = [] - # 使用父类方法读取时间序列数据 - timeseries_data = self.read_ts_xrdataset( - gage_id_lst=object_ids, - t_range=t_range, - var_lst=var_lst_tggnn, - time_units=time_units, + for node in basin_names: + in_degree = dg.in_degree(node) + if in_degree == 0: + source_nodes.append(node) + else: + confluence_nodes.append(node) + + node_mask = [node in confluence_nodes for node in basin_names] + + print(f"源头节点 ({len(source_nodes)}个): {source_nodes}") + print(f"汇流节点 ({len(confluence_nodes)}个): {confluence_nodes}") + + return source_nodes, confluence_nodes, node_mask + + # ===== Helpers for intermediate directory support ===== + def _get_intermediate_root_dir(self): + # 优先使用 set_data_source_describe 中的路径 + return self.data_source_description.get( + "INTERMEDIATE_ROOT_DIR", os.path.join(self.data_source_dir, "intermediate") ) - # 读取LSTM预测数据 - lstm_data = self.read_lstm_predictions( - object_ids=object_ids, t_range_list=t_range + def _where_intermediate_ts_dir(self): + # 已在 set_data_source_describe 中计算，直接提供 + inter_dirs = self.data_source_description.get("INTERMEDIATE_TS_DIRS") + if inter_dirs is not None: + return inter_dirs + ts_dir = os.path.join(self._get_intermediate_root_dir(), "timeseries") + return self._where_ts_dir(ts_dir) + + def _get_intermediate_ts_dir(self, time_unit): + ts_dirs = self._where_intermediate_ts_dir() + # 选择匹配 time_unit 的目录（不附加版本后缀，目录名应与 time_unit 一致） + try: + base_dir = next( + dir_path for dir_path in ts_dirs if time_unit == os.path.basename(dir_path) + ) + except StopIteration: + raise FileNotFoundError( + f"未找到 intermediate/timeseries 下的 {time_unit} 目录，请检查数据结构" + ) + return base_dir + + def _get_intermediate_attr_file(self): + # 使用描述中的路径，避免重复拼接 + return self.data_source_description.get( + "INTERMEDIATE_ATTR_FILE", + os.path.join(self._get_intermediate_root_dir(), "attributes", "attributes.csv"), ) - return timeseries_data, lstm_data + def _get_intermediate_shape_file(self): + return self.data_source_description.get( + "INTERMEDIATE_SHAPE_FILE", + os.path.join(self._get_intermediate_root_dir(), "shapes", "basins.shp"), + ) - def read_node_attributes(self, object_ids=None, selected_attrs=None): - """ - 读取节点静态属性数据 + def _get_intermediate_basinoutlets_file(self): + return self.data_source_description.get( + "INTERMEDIATE_BASINOUTLETS_FILE", + os.path.join(self._get_intermediate_root_dir(), "shapes", "basinoutlets.shp"), + ) + + def read_intermediate_timeseries( + self, object_ids=None, t_range_list: list = None, relevant_cols=None, **kwargs + ) -> dict: + """Read timeseries from the intermediate/timeseries directory. Parameters ---------- - object_ids : array-like, optional - 流域ID列表, by default None - selected_attrs : list, optional - 选择的属性列表, by default None + object_ids : list, optional + Basin IDs to read + t_range_list : list, optional + [start_time, end_time] + relevant_cols : list, optional + Variables to read + **kwargs : dict + time_units, start0101_freq, offset_to_utc, start_hour_in_a_day Returns ------- - xr.Dataset - 属性数据集 + dict + {time_unit: np.ndarray [n_basin, n_time, n_var]} """ - if selected_attrs is None: - selected_attrs = ["ele_mt_sav", "area", "p_mean"] - - # 使用父类方法读取属性数据 - attr_data = self.read_attr_xrdataset(var_lst=selected_attrs) - - # 如果指定了object_ids，则筛选数据 - if object_ids is not None: - # 确保object_ids中的所有ID都存在于数据中 - available_basins = attr_data.coords["basin"].values - valid_ids = [ - basin_id for basin_id in object_ids if basin_id in available_basins - ] - if len(valid_ids) != len(object_ids): - missing_ids = set(object_ids) - set(valid_ids) - print(f"警告：以下流域ID在属性数据中不存在: {missing_ids}") + time_units = kwargs.get("time_units", ["1D"]) # default aligns with parent + start0101_freq = kwargs.get("start0101_freq", False) + offset_to_utc = kwargs.get("offset_to_utc", self.offset_to_utc) + start_hour_in_a_day = kwargs.get("start_hour_in_a_day", 2) - if valid_ids: - attr_data = attr_data.sel(basin=valid_ids) + if object_ids is None or t_range_list is None or relevant_cols is None: + raise ValueError("object_ids, t_range_list, relevant_cols 不能为空") - return attr_data + # Validate start_hour_in_a_day range + if not isinstance(start_hour_in_a_day, int) or not (0 <= start_hour_in_a_day <= 23): + raise ValueError( + f"start_hour_in_a_day must be an integer between 0 and 23, got {start_hour_in_a_day}" + ) - def gen_nx_graph(self, basin_id, object_ids=None): - """ - 生成网络图，确保节点索引与数据中basin顺序一致 + results = {} - Parameters - ---------- - basin_id : str - 流域ID - object_ids : array-like, optional - 对象ID列表，如果为None则使用所有可用的流域ID, by default None + for time_unit in time_units: + if offset_to_utc: + # Prefer intermediate basinoutlets.shp if available; otherwise fall back to base + inter_basinoutlets = self._get_intermediate_basinoutlets_file() + base_basinoutlets = os.path.join( + self.data_source_description["SHAPE_DIR"], "basinoutlets.shp" + ) + basinoutlets_path = ( + inter_basinoutlets if os.path.exists(inter_basinoutlets) else base_basinoutlets + ) + try: + offset_dict = calculate_basin_offsets(basinoutlets_path) + except FileNotFoundError as e: + raise FileNotFoundError( + f"basinoutlets.shp not found in {basinoutlets_path}." + ) from e - Returns - ------- - tuple - (NetworkX图对象, 边列表, 节点映射字典, 有效的对象ID列表) - """ - basin_path_lists = self.graph_dict.get(basin_id) - if basin_path_lists is None: - raise ValueError(f"未找到流域 {basin_id} 的图网络结构") + # Resolve intermediate ts dir for this time unit + ts_dir = self._get_intermediate_ts_dir(time_unit) - # 构建有向图 - dg = nx.DiGraph() - for path in basin_path_lists: - nx.add_path(dg, path) + # Generate time range + if start0101_freq: + t_range = generate_start0101_time_range( + start_time=t_range_list[0], end_time=t_range_list[-1], freq=time_unit + ) + else: + t_range = pd.date_range(start=t_range_list[0], end=t_range_list[-1], freq=time_unit) - # 获取图中所有节点 - graph_nodes = set(dg.nodes) + nt = len(t_range) + x = np.full([len(object_ids), nt, len(relevant_cols)], np.nan) - # 如果没有指定object_ids，使用图中所有节点 - if object_ids is None: - # 从数据中获取可用的流域ID - available_basins = self.read_object_ids() - # 取图中节点与可用数据的交集 - valid_object_ids = [ - basin_id for basin_id in available_basins if basin_id in graph_nodes - ] - else: - # 检查指定的object_ids是否在图中存在 - object_ids_list = list(object_ids) - valid_object_ids = [ - basin_id for basin_id in object_ids_list if basin_id in graph_nodes - ] + # Flag to check data alignment only once + data_alignment_checked = False - # 报告不在图中的节点 - missing_in_graph = set(object_ids_list) - graph_nodes - if missing_in_graph: - print(f"警告：以下流域ID在图网络结构中不存在: {missing_in_graph}") + for k in tqdm(range(len(object_ids)), desc=f"Reading intermediate timeseries with {time_unit}"): + ts_file = os.path.join(ts_dir, object_ids[k] + ".csv") + if "s3://" in ts_file: + with conf.FS.open(ts_file, mode="rb") as f: + ts_data = pd.read_csv(f, engine="c") + else: + ts_data = pd.read_csv(ts_file, engine="c") - # 创建节点映射（节点名称到索引的映射） - node_mapping = {node_id: idx for idx, node_id in enumerate(valid_object_ids)} + date = pd.to_datetime(ts_data["time"]).values + if offset_to_utc: + date = date - np.timedelta64(offset_dict[object_ids[k]], "h") - # 生成边列表（使用索引） - edges = [] - for u, v in dg.edges: - if u in node_mapping and v in node_mapping: - edges.append((node_mapping[u], node_mapping[v])) + # Validate data alignment with start_hour_in_a_day (only check once) + if not data_alignment_checked: + self._validate_time_alignment(date, time_unit, start_hour_in_a_day, object_ids[k]) + data_alignment_checked = True - return dg, edges, node_mapping, valid_object_ids + [_, ind1, ind2] = np.intersect1d(date, t_range, return_indices=True) - def classify_nodes(dg, basin_names): - """ - 分类节点为源头节点和汇流节点 + for j in range(len(relevant_cols)): + tmp_ = self._read_timeseries_1basin1var(ts_data, relevant_cols[j]) + x[k, ind2, j] = tmp_[ind1] - Args: - dg: NetworkX有向图 - basin_names: 节点名称列表 + results[time_unit] = x - Returns: - source_nodes: 源头节点列表（没有入度的节点） - confluence_nodes: 汇流节点列表（有入度的节点） - node_mask: 布尔掩码，True表示汇流节点，False表示源头节点 - """ - source_nodes = [] - confluence_nodes = [] + return results - for node in basin_names: - # 检查节点的入度 - in_degree = dg.in_degree(node) - if in_degree == 0: - source_nodes.append(node) + def cache_intermediate_timeseries_xrdataset(self, **kwargs): + """Cache intermediate timeseries into NetCDF files per time unit, similar to parent cache_timeseries_xrdataset.""" + batchsize = kwargs.get("batchsize", 100) + time_units = kwargs.get("time_units", self.time_unit) or ["1D"] + start0101_freq = kwargs.get("start0101_freq", False) + offset_to_utc = kwargs.get("offset_to_utc", self.offset_to_utc) + start_hour_in_a_day = kwargs.get("start_hour_in_a_day", 2) + + # Validate start_hour_in_a_day + if not isinstance(start_hour_in_a_day, int) or not (0 <= start_hour_in_a_day <= 23): + raise ValueError( + f"start_hour_in_a_day must be an integer between 0 and 23, got {start_hour_in_a_day}" + ) + + variables = self.get_timeseries_cols() # assume units_info identical and stored under base timeseries + basins = self.camels_sites["basin_id"].values + + # Define the generator function for batching + def data_generator(basins_, batch_size): + for i in range(0, len(basins_), batch_size): + yield basins_[i : i + batch_size] + + for time_unit in time_units: + # Set default cache time range + if self.trange4cache is None: + if time_unit != "3h": + self.trange4cache = ["1960-01-01", "2024-12-31"] + else: + # Calculate dynamic end hour for 3h intervals + hours_in_day = list(range(start_hour_in_a_day, 24, 3)) + start_hour = str(start_hour_in_a_day).zfill(2) + end_hour = str(hours_in_day[-1] if hours_in_day else 23).zfill(2) + self.trange4cache = [f"1960-01-01 {start_hour}", f"2024-12-31 {end_hour}"] + + # Build times for coordinates + if start0101_freq: + times = ( + generate_start0101_time_range( + start_time=self.trange4cache[0], end_time=self.trange4cache[-1], freq=time_unit + ) + .strftime("%Y-%m-%d %H:%M:%S") + .tolist() + ) else: - confluence_nodes.append(node) + times = ( + pd.date_range(start=self.trange4cache[0], end=self.trange4cache[-1], freq=time_unit) + .strftime("%Y-%m-%d %H:%M:%S") + .tolist() + ) - # 创建节点掩码：True表示汇流节点（需要训练），False表示源头节点（不训练） - node_mask = [] - for node in basin_names: - node_mask.append(node in confluence_nodes) + # Retrieve units_info for this time unit from base units files + unit_file = next( + file for file in self.data_source_description["UNIT_FILES"] if time_unit in file + ) + if "s3://" in unit_file: + with conf.FS.open(unit_file, mode="rb") as fp: + units_info = json.load(fp) + else: + units_info = hydro_file.unserialize_json(unit_file) - print(f"源头节点 ({len(source_nodes)}个): {source_nodes}") - print(f"汇流节点 ({len(confluence_nodes)}个): {confluence_nodes}") + for basin_batch in data_generator(basins, batchsize): + data = self.read_intermediate_timeseries( + object_ids=basin_batch, + t_range_list=self.trange4cache, + relevant_cols=variables[time_unit], + time_units=[time_unit], + start0101_freq=start0101_freq, + offset_to_utc=offset_to_utc, + start_hour_in_a_day=start_hour_in_a_day, + ) - return source_nodes, confluence_nodes, node_mask + dataset = xr.Dataset( + data_vars={ + variables[time_unit][i]: ( + ["basin", "time"], + data[time_unit][:, :, i], + {"units": units_info[variables[time_unit][i]]}, + ) + for i in range(len(variables[time_unit])) + }, + coords={ + "basin": basin_batch, + "time": pd.to_datetime(times), + }, + ) + + prefix_ = self._get_ts_file_prefix_(f"{self.dataset_name}_intermediate", self.version) + batch_file_path = os.path.join( + CACHE_DIR, + f"{prefix_}timeseries_{time_unit}_batch_{basin_batch[0]}_{basin_batch[-1]}.nc", + ) + dataset.to_netcdf(batch_file_path) + + # Release memory + del dataset + del data def read_ts_xrdataset( self, @@ -2298,15 +2410,325 @@ def read_ts_xrdataset( **kwargs, ) -> dict: """ - Read the time series data from the data source. + 读取时间序列数据，兼容基础目录与 intermediate 目录的缓存读取。 - TODO: Implement this function. + - 默认沿用父类逻辑读取基础缓存； + - 当设置 `prefer_intermediate=True` 或存在 `combine=True` 时，读取/生成 intermediate 缓存； + - intermediate 缓存以 `dataset_name_intermediate` 作为前缀独立保存。 """ - pass + if var_lst is None: + return None + + time_units = kwargs.get("time_units", self.time_unit) + recache = kwargs.get("recache", False) + prefer_intermediate = kwargs.get("prefer_intermediate", False) + combine = kwargs.get("combine", False) + # 是否需要注入 LSTM 预测变量 + want_lstm = any(v == "lstm_pred" for v in (var_lst or [])) + # 基础缓存读取所需的变量（剔除 lstm_pred，避免父类校验失败） + base_vars = [v for v in (var_lst or []) if v != "lstm_pred"] + + # 读取基础缓存数据（剔除 lstm_pred） + base_by_time = super().read_ts_xrdataset( + gage_id_lst=gage_id_lst, t_range=t_range, var_lst=base_vars, **kwargs + ) + + # 为基础数据集注入来源标识 + for time_unit in time_units: + ds_base = base_by_time.get(time_unit) + if isinstance(ds_base, xr.Dataset) and ds_base.dims.get("basin", 0) > 0: + source_arr = xr.DataArray( + data=np.array(["base"] * len(ds_base["basin"]), dtype=object), + dims=["basin"], + coords={"basin": ds_base["basin"]}, + name="source", + ) + base_by_time[time_unit] = ds_base.assign(source=source_arr) + + # Prepare intermediate datasets + inter_by_time = {} + prefix_inter = self._get_ts_file_prefix_(f"{self.dataset_name}_intermediate", self.version) + + # Only build/read intermediate caches when needed + if prefer_intermediate or combine: + for time_unit in time_units: + batch_files_inter = self._get_batch_files(prefix_inter, time_unit) + if not batch_files_inter or recache: + # Build intermediate caches for this time unit + self.cache_intermediate_timeseries_xrdataset( + time_units=[time_unit], + batchsize=kwargs.get("batchsize", 100), + start0101_freq=kwargs.get("start0101_freq", False), + offset_to_utc=kwargs.get("offset_to_utc", self.offset_to_utc), + start_hour_in_a_day=kwargs.get("start_hour_in_a_day", 2), + ) + batch_files_inter = self._get_batch_files(prefix_inter, time_unit) + + selected_datasets = [] + for batch_file in batch_files_inter: + ds = xr.open_dataset(batch_file) + all_vars = ds.data_vars + if base_vars and any(var not in ds.variables for var in base_vars): + ds.close() + raise ValueError(f"var_lst must all be in {all_vars}") + valid_gage_ids = ( + [gid for gid in (gage_id_lst or []) if gid in ds["basin"].values] + if gage_id_lst is not None + else ds["basin"].values.tolist() + ) + if len(valid_gage_ids) > 0: + ds_selected = ds[base_vars].sel( + basin=valid_gage_ids, time=slice(t_range[0], t_range[1]) + ) + selected_datasets.append(ds_selected) + ds.close() + + if selected_datasets: + inter_ds = xr.concat(selected_datasets, dim="basin").sortby("basin") + # 注入来源标识 + source_arr = xr.DataArray( + data=np.array(["intermediate"] * len(inter_ds["basin"]), dtype=object), + dims=["basin"], + coords={"basin": inter_ds["basin"]}, + name="source", + ) + inter_by_time[time_unit] = inter_ds.assign(source=source_arr) + else: + inter_by_time[time_unit] = xr.Dataset() + + # 如果需要，将 LSTM 预测数据注入基础数据集中 + if want_lstm: + lstm_pred_file = self.data_source_description.get("LSTM_PRED_FILE") + + # 统一存在性校验：优先使用描述中的标志位，其次实时检查 + exists_lstm = False + if lstm_pred_file is not None: + if "s3://" in lstm_pred_file: + try: + exists_lstm = conf.FS.exists(lstm_pred_file) + except Exception: + exists_lstm = False + else: + exists_lstm = os.path.exists(lstm_pred_file) + + if not lstm_pred_file or not exists_lstm: + raise FileNotFoundError( + f"lstmpred.nc 未找到：{lstm_pred_file}。请确保在数据目录下提供 lstmpred.nc。" + ) + # 打开 LSTM 预测文件并读取 streamflow 变量 + if "s3://" in lstm_pred_file: + with conf.FS.open(lstm_pred_file, mode="rb") as f: + lstm_ds_all = xr.open_dataset(f) + else: + lstm_ds_all = xr.open_dataset(lstm_pred_file) + + # 处理每个 time_unit 的注入（按需重采样） + for time_unit in time_units: + ds_base = base_by_time.get(time_unit, xr.Dataset()) + # 如果基础数据集为空，跳过注入（避免维度不匹配） + if not isinstance(ds_base, xr.Dataset) or ds_base.dims.get("basin", 0) == 0: + continue + + # 选择有效的 basin 和时间范围 + valid_ids = ( + [gid for gid in (gage_id_lst or []) if gid in lstm_ds_all["basin"].values] + if gage_id_lst is not None + else ds_base["basin"].values.tolist() + ) + if len(valid_ids) == 0: + # 无有效 basin 时不注入 + continue + + lstm_sel = lstm_ds_all.sel(basin=valid_ids) + if t_range is not None: + lstm_sel = lstm_sel.sel(time=slice(t_range[0], t_range[1])) + + # 根据 time_unit 重采样（若需要），默认使用均值 + # 将 'h' 替换为大写 'H' 以兼容 pandas resample + resample_freq = time_unit.replace("h", "H") + try: + lstm_resampled = lstm_sel.resample(time=resample_freq).mean() + except Exception: + # 如果 time_unit 无法用于重采样，则直接使用原始频率 + lstm_resampled = lstm_sel + + # 重命名变量为 lstm_pred，仅注入到基础数据集中 + if "streamflow" not in lstm_resampled.data_vars: + lstm_ds_all.close() + raise ValueError("lstmpred.nc 中缺少 'streamflow' 变量，无法注入 'lstm_pred'") + + lstm_var = lstm_resampled["streamflow"].rename("lstm_pred") + # 对齐时间范围 + if t_range is not None: + lstm_var = lstm_var.sel(time=slice(t_range[0], t_range[1])) + + # 注入变量（自动按坐标对齐） + base_by_time[time_unit] = ds_base.assign(lstm_pred=lstm_var) + + lstm_ds_all.close() + + # Decide return + if prefer_intermediate: + return inter_by_time + if combine: + combined = {} + for time_unit in time_units: + base_ds = base_by_time.get(time_unit, xr.Dataset()) + inter_ds = inter_by_time.get(time_unit, xr.Dataset()) + if getattr(base_ds, "dims", {}).get("basin", 0) == 0: + combined[time_unit] = inter_ds + elif getattr(inter_ds, "dims", {}).get("basin", 0) == 0: + combined[time_unit] = base_ds + else: + combined[time_unit] = xr.concat([base_ds, inter_ds], dim="basin").sortby("basin") + return combined + return base_by_time def read_attr_xrdataset(self, gage_id_lst=None, var_lst=None, **kwargs): - """Read the attribute data from the data source. + """读取属性数据，兼容基础目录与 intermediate 目录的缓存。 - TODO: Implement this function. + - 默认沿用父类逻辑读取基础缓存； + - 当设置 `prefer_intermediate=True` 或 `combine=True` 时，读取/生成 intermediate 缓存； + - intermediate 缓存以 `dataset_name_intermediate` 作为前缀独立保存。 """ - pass + if var_lst is None or len(var_lst) == 0: + return None + + recache = kwargs.get("recache", False) + prefer_intermediate = kwargs.get("prefer_intermediate", False) + combine = kwargs.get("combine", False) + + # Base attributes (handle None gage_id_lst gracefully) + dataset_name = self.dataset_name + prefix_base = "" if dataset_name is None else dataset_name + "_" + recache_base = kwargs.get("recache", False) + base_attr_path = os.path.join(CACHE_DIR, f"{prefix_base}attributes.nc") + if (not os.path.exists(base_attr_path)) or recache_base: + self.cache_attributes_xrdataset() + base_all = xr.open_dataset(base_attr_path) + if any(var not in base_all.variables for var in var_lst): + base_all.close() + raise ValueError(f"var_lst must all be in {list(base_all.data_vars)}") + base_ds = base_all[var_lst] + if gage_id_lst is not None: + base_ds = base_ds.sel(basin=gage_id_lst) + # 注入来源标识 + if isinstance(base_ds, xr.Dataset) and base_ds.dims.get("basin", 0) > 0: + source_arr_base = xr.DataArray( + data=np.array(["base"] * len(base_ds["basin"]), dtype=object), + dims=["basin"], + coords={"basin": base_ds["basin"]}, + name="source", + ) + base_ds = base_ds.assign(source=source_arr_base) + base_all.close() + + # Prepare intermediate cache file + prefix_inter = "" if self.dataset_name is None else f"{self.dataset_name}_intermediate_" + inter_attr_file = os.path.join(CACHE_DIR, f"{prefix_inter}attributes.nc") + + # Build intermediate cache when needed + inter_ds = None + if prefer_intermediate or combine: + if (not os.path.exists(inter_attr_file)) or recache: + try: + self.cache_intermediate_attributes_xrdataset() + except FileNotFoundError: + # If intermediate sources are missing, skip + pass + if os.path.exists(inter_attr_file): + inter_all = xr.open_dataset(inter_attr_file) + if any(var not in inter_all.variables for var in var_lst): + inter_all.close() + raise ValueError(f"var_lst must all be in {list(inter_all.data_vars)}") + inter_ds = inter_all[var_lst] + if gage_id_lst is not None: + inter_ds = inter_ds.sel(basin=gage_id_lst) + # 注入来源标识 + if isinstance(inter_ds, xr.Dataset) and inter_ds.dims.get("basin", 0) > 0: + source_arr_inter = xr.DataArray( + data=np.array(["intermediate"] * len(inter_ds["basin"]), dtype=object), + dims=["basin"], + coords={"basin": inter_ds["basin"]}, + name="source", + ) + inter_ds = inter_ds.assign(source=source_arr_inter) + inter_all.close() + + if prefer_intermediate: + return inter_ds + if combine and inter_ds is not None and base_ds is not None: + # Concatenate along basin dimension; sort for consistency + return xr.concat([base_ds, inter_ds], dim="basin").sortby("basin") + return base_ds if base_ds is not None else inter_ds + + def cache_intermediate_attributes_xrdataset(self): + """Convert intermediate attributes to a single dataset and cache as NetCDF.""" + import pint_xarray # noqa: F401 + + # Resolve shape file (prefer intermediate, else base) + shape_file_inter = self._get_intermediate_shape_file() + if os.path.exists(shape_file_inter): + if "s3://" in shape_file_inter: + with conf.FS.open(shape_file_inter, mode="rb") as f: + shape = gpd.read_file(f) + else: + shape = gpd.read_file(shape_file_inter) + else: + shape_dir = os.path.join(self.data_source_description["SHAPE_DIR"], "basins.shp") + if "s3://" in shape_dir: + with conf.FS.open(shape_dir, mode="rb") as f: + shape = gpd.read_file(f) + else: + shape = gpd.read_file(shape_dir) + + df_area = cal_area_from_shp(shape) + df_area.set_index("basin_id", inplace=True) + + # Intermediate attributes file + attr_file = self._get_intermediate_attr_file() + if "s3://" in attr_file: + with conf.FS.open(attr_file, mode="rb") as f: + df_attr = pd.read_csv(f, dtype={"basin_id": str}) + else: + df_attr = pd.read_csv(attr_file, dtype={"basin_id": str}) + + df_attr.set_index("basin_id", inplace=True) + df_attr = df_attr.join(df_area, how="left") + + # Units mapping from base units_info (shared structure) + if "s3://" in self.data_source_description["UNIT_FILES"][0]: + with conf.FS.open(self.data_source_description["UNIT_FILES"][0], mode="rb") as fp: + units_dict = json.load(fp) + else: + units_dict = hydro_file.unserialize_json(self.data_source_description["UNIT_FILES"][0]) + units_dict["shp_area"] = "km^2" + units_dict["area"] = "km^2" + + # Convert object columns to categorical codes with mapping + categorical_mappings = {} + for column in df_attr.columns: + if df_attr[column].dtype == "object": + df_attr[column] = df_attr[column].astype("category") + categorical_mappings[column] = dict(enumerate(df_attr[column].cat.categories)) + df_attr[column] = df_attr[column].cat.codes + + ds = xr.Dataset() + for column in df_attr.columns: + attrs = {"units": units_dict.get(column, "unknown")} + if column in categorical_mappings: + attrs["category_mapping"] = categorical_mappings[column] + ds[column] = xr.DataArray( + data=df_attr[column].values, + dims=["basin"], + coords={"basin": df_attr.index.values.astype(str)}, + attrs=attrs, + ) + + # Convert categorical mapping dicts to strings + for column in ds.data_vars: + if "category_mapping" in ds[column].attrs: + ds[column].attrs["category_mapping"] = str(ds[column].attrs["category_mapping"]) + + prefix_ = "" if self.dataset_name is None else f"{self.dataset_name}_intermediate_" + ds.to_netcdf(os.path.join(CACHE_DIR, f"{prefix_}attributes.nc")) diff --git a/tests/test_tg_hydro_datasource.py b/tests/test_tg_hydro_datasource.py index d2f2a09..a7af500 100644 --- a/tests/test_tg_hydro_datasource.py +++ b/tests/test_tg_hydro_datasource.py @@ -1,43 +1,33 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ -测试TgHydroDatasource类的功能 +重构测试：验证 TgHydroDatasource 的 read_ts_xrdataset 与 read_attr_xrdataset 输出形状与变量 """ import os -import sys import json import numpy as np import pandas as pd +import pytest import xarray as xr import geopandas as gpd from shapely.geometry import Polygon -from pathlib import Path - -# 添加项目路径到sys.path -project_root = Path(__file__).parent -sys.path.insert(0, str(project_root)) +import hydrodatasource.configs.config as conf from hydrodatasource.reader.data_source import TgHydroDatasource -def create_test_data_structure(test_data_dir): - """创建测试数据目录结构""" - print("创建测试数据目录结构...") - - # 创建目录结构 - dirs_to_create = ["attributes", "timeseries/1D", "shapes", "tggnn"] - - for dir_name in dirs_to_create: - dir_path = os.path.join(test_data_dir, dir_name) - os.makedirs(dir_path, exist_ok=True) - print(f"创建目录: {dir_path}") +def _create_dirs(root, dataset_name): + base = os.path.join(root, dataset_name) + os.makedirs(os.path.join(base, "attributes"), exist_ok=True) + os.makedirs(os.path.join(base, "timeseries", "1D"), exist_ok=True) + os.makedirs(os.path.join(base, "shapes"), exist_ok=True) + os.makedirs(os.path.join(base, "tggnn"), exist_ok=True) + os.makedirs(os.path.join(base, "intermediate"), exist_ok=True) + return base -def create_test_graph_dict(test_data_dir): - """创建测试用的graph_dict.json文件""" - print("创建测试graph_dict.json文件...") - +def _create_graph_dict(base): graph_dict = { "test_basin": [ ["station_1", "station_2", "station_3"], @@ -45,20 +35,15 @@ def create_test_graph_dict(test_data_dir): ["station_5", "station_6", "station_3"], ] } - - graph_dict_file = os.path.join(test_data_dir, "tggnn", "graph_dict.json") + # TgHydroDatasource 读取路径为 /intermediate/graph_dict.json + graph_dict_file = os.path.join(base, "intermediate", "graph_dict.json") with open(graph_dict_file, "w", encoding="utf-8") as f: json.dump(graph_dict, f, ensure_ascii=False, indent=2) + return graph_dict_file - print(f"创建文件: {graph_dict_file}") - return graph_dict - -def create_test_lstm_predictions(test_data_dir): - """创建测试用的lstmpred.nc文件""" - print("创建测试lstmpred.nc文件...") - - # 创建测试数据 +def _create_lstm_predictions(base): + # Hourly streamflow for resampling to 1D time_range = pd.date_range("2020-01-01", "2020-01-10", freq="H") basin_names = [ "station_1", @@ -68,30 +53,26 @@ def create_test_lstm_predictions(test_data_dir): "station_5", "station_6", ] - - # 生成随机流量数据 np.random.seed(42) streamflow_data = np.random.rand(len(basin_names), len(time_range)) * 100 - - # 创建xarray Dataset lstm_dataset = xr.Dataset( {"streamflow": (["basin", "time"], streamflow_data)}, coords={"basin": basin_names, "time": time_range}, ) - - lstm_pred_file = os.path.join(test_data_dir, "tggnn", "lstmpred.nc") + # 写入到数据源目录（便于调试和留存） + lstm_pred_file = os.path.join(base, "tggnn", "lstmpred.nc") + os.makedirs(os.path.dirname(lstm_pred_file), exist_ok=True) lstm_dataset.to_netcdf(lstm_pred_file) - print(f"创建文件: {lstm_pred_file}") - return lstm_dataset - + # 同步写入到缓存目录，满足 TgHydroDatasource 的路径约定 + os.makedirs(conf.CACHE_DIR, exist_ok=True) + lstm_dataset.to_netcdf(os.path.join(conf.CACHE_DIR, "lstmpred.nc")) + return lstm_pred_file -def create_test_timeseries(test_data_dir): - """创建测试用的时间序列数据""" - print("创建测试时间序列数据...") - # 创建测试数据 - time_range = pd.date_range("2020-01-01", "2020-01-10", freq="H") +def _create_timeseries_csv(base): + # Daily timeseries CSV per basin, as expected by reader + time_range = pd.date_range("2020-01-01", "2020-01-10", freq="1D") basin_names = [ "station_1", "station_2", @@ -100,34 +81,19 @@ def create_test_timeseries(test_data_dir): "station_5", "station_6", ] + np.random.seed(123) + for name in basin_names: + df = pd.DataFrame( + { + "time": time_range, + "total_precipitation_hourly": np.random.rand(len(time_range)) * 10, + "streamflow": np.random.rand(len(time_range)) * 50, + } + ) + df.to_csv(os.path.join(base, "timeseries", "1D", f"{name}.csv"), index=False) - # 生成随机数据 - np.random.seed(42) - precipitation_data = np.random.rand(len(basin_names), len(time_range)) * 10 - streamflow_data = np.random.rand(len(basin_names), len(time_range)) * 50 - - # 创建xarray Dataset - timeseries_dataset = xr.Dataset( - { - "total_precipitation_hourly": (["basin", "time"], precipitation_data), - "streamflow": (["basin", "time"], streamflow_data), - }, - coords={"basin": basin_names, "time": time_range}, - ) - - timeseries_file = os.path.join( - test_data_dir, "timeseries", "1D", "timeseries_1D.nc" - ) - timeseries_dataset.to_netcdf(timeseries_file) - - print(f"创建文件: {timeseries_file}") - return timeseries_dataset - - -def create_test_shapes(test_data_dir): - """创建测试用的形状文件""" - print("创建测试形状文件...") +def _create_shapes(base): basin_names = [ "station_1", "station_2", @@ -136,13 +102,10 @@ def create_test_shapes(test_data_dir): "station_5", "station_6", ] - - # 创建简单的多边形几何 geometries = [] - for i, basin_name in enumerate(basin_names): - # 创建简单的矩形多边形 (使用经纬度坐标) - x_offset = 100 + i * 0.1 # 经度 - y_offset = 30 + i * 0.1 # 纬度 + for i, _ in enumerate(basin_names): + x_offset = 100 + i * 0.1 + y_offset = 30 + i * 0.1 polygon = Polygon( [ (x_offset, y_offset), @@ -152,45 +115,25 @@ def create_test_shapes(test_data_dir): ] ) geometries.append(polygon) + gdf = gpd.GeoDataFrame({"BASIN_ID": basin_names, "geometry": geometries}, crs="EPSG:4326") + gdf.to_file(os.path.join(base, "shapes", "basins.shp")) - # 创建GeoDataFrame并设置CRS为WGS84 - gdf = gpd.GeoDataFrame( - {"BASIN_ID": basin_names, "geometry": geometries}, # 使用BASIN_ID列名 - crs="EPSG:4326", - ) # WGS84坐标系 - - shapes_file = os.path.join(test_data_dir, "shapes", "basins.shp") - gdf.to_file(shapes_file) - - print(f"创建文件: {shapes_file}") - return gdf - -def create_test_unit_info(test_data_dir): - """创建测试用的单位信息文件""" - print("创建测试单位信息文件...") - - # 创建单位信息字典 +def _create_unit_info(base): unit_info = { "ele_mt_sav": "m", - "area": "km2", + "area": "km^2", "p_mean": "mm", "total_precipitation_hourly": "mm/h", "streamflow": "mm/d", } - - unit_file = os.path.join(test_data_dir, "timeseries", "1D_units_info.json") + unit_file = os.path.join(base, "timeseries", "1D_units_info.json") with open(unit_file, "w", encoding="utf-8") as f: json.dump(unit_info, f, ensure_ascii=False, indent=2) - - print(f"创建文件: {unit_file}") - return unit_info + return unit_file -def create_test_attributes(test_data_dir): - """创建测试用的属性数据""" - print("创建测试属性数据...") - +def _create_attributes(base): basin_names = [ "station_1", "station_2", @@ -199,174 +142,239 @@ def create_test_attributes(test_data_dir): "station_5", "station_6", ] - - # 生成随机属性数据 - np.random.seed(42) - ele_mt_sav = np.random.rand(len(basin_names)) * 1000 + 500 # 海拔 - area = np.random.rand(len(basin_names)) * 5000 + 1000 # 面积 - p_mean = np.random.rand(len(basin_names)) * 2000 + 800 # 平均降水 - - # 创建DataFrame并保存为CSV - attributes_df = pd.DataFrame( + np.random.seed(7) + df = pd.DataFrame( { "basin_id": basin_names, - "ele_mt_sav": ele_mt_sav, - "area": area, - "p_mean": p_mean, + "ele_mt_sav": np.random.rand(len(basin_names)) * 1000 + 500, + "area": np.random.rand(len(basin_names)) * 5000 + 1000, + "p_mean": np.random.rand(len(basin_names)) * 2000 + 800, } ) + df.to_csv(os.path.join(base, "attributes", "attributes.csv"), index=False) + + +@pytest.fixture +def tg_dataset(tmp_path): + dataset_name = "test_tg_dataset" + base = _create_dirs(tmp_path, dataset_name) + _create_graph_dict(base) + _create_lstm_predictions(base) + _create_timeseries_csv(base) + _create_shapes(base) + _create_unit_info(base) + _create_attributes(base) + + ds = TgHydroDatasource(data_path=str(tmp_path), dataset_name=dataset_name, time_unit=["1D"]) + return ds + + +def test_read_ts_xrdataset_shapes_and_vars(tg_dataset): + basins = ["station_1", "station_2", "station_3"] + t_range = ["2020-01-01", "2020-01-10"] + var_lst = ["discharge", "total_precipitation_hourly", "streamflow", "lstm_pred"] + + # 注意：父类缓存生成只会包含 CSV 的列；这里包含不存在的 "discharge" 会被过滤或触发校验 + # 为避免报错，先限制为 CSV 中存在的列 + lstm_pred + var_lst = ["total_precipitation_hourly", "streamflow", "lstm_pred"] + + ds_map = tg_dataset.read_ts_xrdataset( + gage_id_lst=basins, + t_range=t_range, + var_lst=var_lst, + time_units=["1D"], + ) - attributes_file = os.path.join(test_data_dir, "attributes", "attributes.csv") - attributes_df.to_csv(attributes_file, index=False) - - print(f"创建文件: {attributes_file}") - return attributes_df - - -def test_tg_hydro_datasource(): - """测试TgHydroDatasource类""" - print("=" * 60) - print("开始测试TgHydroDatasource类") - print("=" * 60) - - # 设置测试数据目录 - test_data_dir = "/tmp/test_tg_hydro_data" - - try: - # 1. 创建测试数据 - create_test_data_structure(test_data_dir) - graph_dict = create_test_graph_dict(test_data_dir) - lstm_data = create_test_lstm_predictions(test_data_dir) - timeseries_data = create_test_timeseries(test_data_dir) - shapes_data = create_test_shapes(test_data_dir) - unit_info = create_test_unit_info(test_data_dir) - attributes_data = create_test_attributes(test_data_dir) - - # 2. 初始化TgHydroDatasource - print("\n" + "=" * 40) - print("测试TgHydroDatasource初始化") - print("=" * 40) - - tg_datasource = TgHydroDatasource( - data_path=test_data_dir, time_unit=["1D"], dataset_name="test_tg_dataset" - ) - - print(f"✓ TgHydroDatasource初始化成功") - print(f" 数据源名称: {tg_datasource.get_name()}") - print(f" 数据路径: {tg_datasource.data_source_dir}") - - # 3. 测试数据源描述 - print("\n" + "=" * 40) - print("测试数据源描述") - print("=" * 40) - - data_desc = tg_datasource.data_source_description - print("数据源描述包含的路径:") - for key, value in data_desc.items(): - if "TGGNN" in key or "LSTM" in key or "GRAPH" in key: - print(f" {key}: {value}") - - # 4. 测试图网络结构加载 - print("\n" + "=" * 40) - print("测试图网络结构") - print("=" * 40) - - print(f"✓ 图网络结构加载成功") - print(f" 包含流域: {list(tg_datasource.graph_dict.keys())}") - - # 5. 测试LSTM预测数据读取 - print("\n" + "=" * 40) - print("测试LSTM预测数据读取") - print("=" * 40) - - # 获取实际可用的basin ID - available_basins = tg_datasource.read_object_ids() - test_basins = available_basins[:3] # 使用前三个basin - - lstm_predictions = tg_datasource.read_lstm_predictions( - object_ids=test_basins[:2], t_range_list=["2020-01-01", "2020-01-05"] - ) - - # 5. 测试时间序列数据加载 - print("\n5. 测试时间序列数据加载...") - ts_data, lstm_data = tg_datasource.load_ts_data( - basin_names=test_basins[:2], - t_range=["2020-01-01", "2020-01-05"], - var_lst=["total_precipitation_hourly", "streamflow"], - ) - print(f"✓ LSTM预测数据读取成功") - print(f" 数据形状: {lstm_predictions['streamflow'].shape}") - print( - f" 时间范围: {lstm_predictions.time.values[0]} 到 {lstm_predictions.time.values[-1]}" - ) - print(f"✓ 时间序列数据加载成功") - print(f" 时间序列数据形状: {ts_data.shape}") - print(f" LSTM数据形状: {lstm_data.shape}") - - # 6. 测试节点属性加载 - print("\n" + "=" * 40) - print("测试节点属性加载") - print("=" * 40) - node_attrs = tg_datasource.load_node_attributes( - basin_names=test_basins[:3], selected_attrs=["ele_mt_sav", "area", "p_mean"] - ) - print(f"✓ 节点属性加载成功") - print(f" 属性张量形状: {node_attrs.shape}") - - # 7. 测试图生成 - print("\n" + "=" * 40) - print("测试图生成") - print("=" * 40) - - basin_names_all = [ - "station_1", - "station_2", - "station_3", - "station_4", - "station_5", - "station_6", - ] - dg, edges, node_mapping = tg_datasource.gen_nx_graph( - "test_basin", basin_names_all - ) - print(f"✓ 图生成成功") - print(f" 节点数量: {len(dg.nodes)}") - print(f" 边数量: {len(edges)}") - print(f" 节点映射: {node_mapping}") - - # 8. 测试上游节点查找 - print("\n" + "=" * 40) - print("测试上游节点查找") - print("=" * 40) - - upstream_nodes = tg_datasource.get_upstream_nodes( - "test_basin", "station_3", basin_names_all - ) - print(f"✓ 上游节点查找成功") - print(f" station_3的上游节点索引: {upstream_nodes}") - - print("\n" + "=" * 60) - print("所有测试通过！TgHydroDatasource类工作正常") - print("=" * 60) - - return True - - except Exception as e: - print(f"\n❌ 测试失败: {str(e)}") - import traceback - - traceback.print_exc() - return False - - finally: - # 清理测试数据 - import shutil - - if os.path.exists(test_data_dir): - shutil.rmtree(test_data_dir) - print(f"\n清理测试数据目录: {test_data_dir}") - + assert isinstance(ds_map, dict) + assert "1D" in ds_map + ds = ds_map["1D"] + assert isinstance(ds, xr.Dataset) + + # 直观打印：整体结构与基本信息 + print("\n[TS] Dataset summary:\n", ds) + print("[TS] dims:", ds.dims) + print("[TS] sizes:", dict(ds.sizes)) + print("[TS] vars:", list(ds.data_vars)) + print("[TS] basin sample:", ds["basin"].values[:3] if "basin" in ds.coords else basins) + print("[TS] time sample:", ds["time"].values[:3] if "time" in ds.coords else []) + + # 检查维度（顺序无关，使用 sizes 更稳妥） + assert "basin" in ds.dims and "time" in ds.dims + assert ds.sizes["basin"] == len(basins) + expected_time = pd.date_range(t_range[0], t_range[1], freq="1D") + assert ds.sizes["time"] == len(expected_time) + + # 检查变量存在与形状 + for v in ["streamflow", "total_precipitation_hourly"]: + assert v in ds.data_vars + # 不依赖维度顺序，只需包含两个维度且名称相符 + assert set(ds[v].dims) == {"basin", "time"} + # 直观打印：小样本（3个流域 x 前3天），并确保顺序为 (basin, time) + sample = ds[v].transpose("basin", "time").sel(basin=basins).isel(time=slice(0, 3)) + print(f"[TS] var={v} dims={sample.dims} sizes={dict(sample.sizes)} dtype={sample.dtype}") + print(f"[TS] var={v} sample (first 3 basins x first 3 time):\n", sample.to_pandas()) + + # 检查 lstm_pred 注入 + assert "lstm_pred" in ds.data_vars + assert set(ds["lstm_pred"].dims) == {"basin", "time"} + lstm_sample = ds["lstm_pred"].transpose("basin", "time").sel(basin=basins).isel(time=slice(0, 3)) + print("[TS] var=lstm_pred dims=", lstm_sample.dims, "sizes=", dict(lstm_sample.sizes), "dtype=", lstm_sample.dtype) + print("[TS] var=lstm_pred sample (first 3 basins x first 3 time):\n", lstm_sample.to_pandas()) + + # 检查来源标识 + assert "source" in ds.data_vars + assert ds["source"].dims == ("basin",) + assert set(ds["source"].values.tolist()) == {"base"} + print("[TS] source sample:\n", ds["source"].to_pandas().head()) + + +def test_read_ts_xrdataset_combine_outputs(tg_dataset): + basins = ["station_1", "station_2", "station_3"] + t_range = ["2020-01-01", "2020-01-10"] + var_lst = ["total_precipitation_hourly", "streamflow", "lstm_pred"] + + ds_map = tg_dataset.read_ts_xrdataset( + gage_id_lst=basins, + t_range=t_range, + var_lst=var_lst, + time_units=["1D"], + combine=True, + ) -if __name__ == "__main__": - success = test_tg_hydro_datasource() - sys.exit(0 if success else 1) + assert isinstance(ds_map, dict) and "1D" in ds_map + ds = ds_map["1D"] + assert isinstance(ds, xr.Dataset) + + # 直观打印：整体结构与基本信息（combined） + print("\n[TS-COMBINE] Dataset summary:\n", ds) + print("[TS-COMBINE] dims:", ds.dims) + print("[TS-COMBINE] sizes:", dict(ds.sizes)) + print("[TS-COMBINE] vars:", list(ds.data_vars)) + + # 检查维度与合并后大小 + assert "basin" in ds.dims and "time" in ds.dims + expected_time = pd.date_range(t_range[0], t_range[1], freq="1D") + assert ds.sizes["time"] == len(expected_time) + # 若存在 intermediate，则应为 2 倍；否则为 1 倍 + sources_set = set(ds["source"].values.tolist()) if "source" in ds.data_vars else set() + if {"base", "intermediate"}.issubset(sources_set): + assert ds.sizes["basin"] == len(basins) * 2 + else: + assert ds.sizes["basin"] == len(basins) + + # 检查来源标识包含两类 + assert "source" in ds.data_vars + sources = set(ds["source"].values.tolist()) + # 当 intermediate 不存在时，仅有 base + assert sources in [{"base", "intermediate"}, {"base"}] + print("[TS-COMBINE] source counts:", pd.Series(ds["source"].values).value_counts().to_dict()) + + # 检查变量维度与小样本 + for v in ["streamflow", "total_precipitation_hourly"]: + assert v in ds.data_vars + assert set(ds[v].dims) == {"basin", "time"} + sample = ds[v].transpose("basin", "time").isel(basin=slice(0, 6), time=slice(0, 3)) + print(f"[TS-COMBINE] var={v} dims={sample.dims} sizes={dict(sample.sizes)} dtype={sample.dtype}") + print(f"[TS-COMBINE] var={v} sample (first 6 basins x first 3 time):\n", sample.to_pandas()) + + # lstm_pred 仅注入到 base 部分；intermediate 部分应为空或 NaN + assert "lstm_pred" in ds.data_vars + assert set(ds["lstm_pred"].dims) == {"basin", "time"} + inter_mask = ds["source"].values == "intermediate" + inter_basins = ds["basin"].values[inter_mask] + base_mask = ds["source"].values == "base" + base_basins = ds["basin"].values[base_mask] + # 打印样本 + lstm_base_sample = ds["lstm_pred"].transpose("basin", "time").sel(basin=base_basins).isel(time=slice(0, 3)) + print("[TS-COMBINE] var=lstm_pred base sample:\n", lstm_base_sample.to_pandas().head()) + if len(inter_basins) > 0: + lstm_inter_sample = ds["lstm_pred"].transpose("basin", "time").sel(basin=inter_basins).isel(time=slice(0, 3)) + print("[TS-COMBINE] var=lstm_pred intermediate sample:\n", lstm_inter_sample.to_pandas().head()) + # 检查 intermediate 部分为 NaN + assert np.isnan(lstm_inter_sample.to_numpy()).all() + + + +def test_read_attr_xrdataset_shapes_and_vars(tg_dataset): + basins = ["station_1", "station_2", "station_3"] + var_lst = ["ele_mt_sav", "area", "p_mean"] + + ds = tg_dataset.read_attr_xrdataset(gage_id_lst=basins, var_lst=var_lst) + assert isinstance(ds, xr.Dataset) + + # 直观打印：整体结构与基本信息 + print("\n[ATTR] Dataset summary:\n", ds) + print("[ATTR] dims:", ds.dims) + print("[ATTR] sizes:", dict(ds.sizes)) + print("[ATTR] vars:", list(ds.data_vars)) + print("[ATTR] basin sample:", ds["basin"].values[:3] if "basin" in ds.coords else basins) + + # 检查维度（使用 sizes 更稳妥） + assert "basin" in ds.dims + assert ds.sizes["basin"] == len(basins) + + # 检查变量与单位属性 + for v in var_lst: + assert v in ds.data_vars + # 不依赖维度顺序（这里应为 1 维），直接检查包含 basin 维度 + assert set(ds[v].dims) == {"basin"} + assert ds[v].sizes["basin"] == len(basins) + assert "units" in ds[v].attrs + # 直观打印：小样本（3个流域）与单位 + sample = ds[v].sel(basin=basins) + print(f"[ATTR] var={v} dims={sample.dims} sizes={dict(sample.sizes)} dtype={sample.dtype} units={sample.attrs.get('units')}") + print(f"[ATTR] var={v} sample (first 3 basins):\n", sample.to_pandas()) + + # 检查来源标识 + assert "source" in ds.data_vars + assert ds["source"].dims == ("basin",) + assert set(ds["source"].values.tolist()) == {"base"} + print("[ATTR] source sample:\n", ds["source"].to_pandas().head()) + + +def test_read_attr_xrdataset_combine_outputs(tg_dataset): + basins = ["station_1", "station_2", "station_3"] + var_lst = ["ele_mt_sav", "area", "p_mean"] + + ds = tg_dataset.read_attr_xrdataset(gage_id_lst=basins, var_lst=var_lst, combine=True) + assert isinstance(ds, xr.Dataset) + + # 直观打印：整体结构与基本信息（combined） + print("\n[ATTR-COMBINE] Dataset summary:\n", ds) + print("[ATTR-COMBINE] dims:", ds.dims) + print("[ATTR-COMBINE] sizes:", dict(ds.sizes)) + print("[ATTR-COMBINE] vars:", list(ds.data_vars)) + + # 检查维度与合并后大小 + assert "basin" in ds.dims + sources_set = set(ds["source"].values.tolist()) if "source" in ds.data_vars else set() + if {"base", "intermediate"}.issubset(sources_set): + assert ds.sizes["basin"] == len(basins) * 2 + else: + assert ds.sizes["basin"] == len(basins) + + # 检查变量与单位属性，以及来源类别 + for v in var_lst: + assert v in ds.data_vars + assert set(ds[v].dims) == {"basin"} + if {"base", "intermediate"}.issubset(sources_set): + assert ds[v].sizes["basin"] == len(basins) * 2 + else: + assert ds[v].sizes["basin"] == len(basins) + assert "units" in ds[v].attrs + # 打印样本（前 6 basins） + sample = ds[v].isel(basin=slice(0, 6)) + print(f"[ATTR-COMBINE] var={v} dims={sample.dims} sizes={dict(sample.sizes)} dtype={sample.dtype} units={sample.attrs.get('units')}") + print(f"[ATTR-COMBINE] var={v} sample (first 6 basins):\n", sample.to_pandas()) + + # 检查来源标识 + assert "source" in ds.data_vars + sources = set(ds["source"].values.tolist()) + assert sources in [{"base", "intermediate"}, {"base"}] + print("[ATTR-COMBINE] source counts:", pd.Series(ds["source"].values).value_counts().to_dict()) + + +def test_description_has_lstm_file(tg_dataset): + desc = tg_dataset.data_source_description + assert "LSTM_PRED_FILE" in desc + assert os.path.exists(desc["LSTM_PRED_FILE"]) # 路径应指向缓存目录下的 lstmpred.nc From 84f851c4217dcd832eb17cfbd4d169371c0829da Mon Sep 17 00:00:00 2001 From: zhuanglaihong Date: Wed, 17 Dec 2025 19:20:11 +0800 Subject: [PATCH 10/34] add floodeventdatasource with score --- hydrodatasource/reader/data_source.py | 2 +- hydrodatasource/reader/floodevent.py | 277 ++++++++++++++++++++------ 2 files changed, 216 insertions(+), 63 deletions(-) diff --git a/hydrodatasource/reader/data_source.py b/hydrodatasource/reader/data_source.py index 71c00bc..5c7f48e 100644 --- a/hydrodatasource/reader/data_source.py +++ b/hydrodatasource/reader/data_source.py @@ -747,7 +747,7 @@ def read_ts_xrdataset( selected_datasets.append(ds_selected) ds.close() # Close the dataset to free memory - + print(f"Successfully read from data cache: {batch_file}") # If any datasets were selected, concatenate them along the 'basin' dimension if selected_datasets: # NOTE: the chosen part must be sorted by basin, or there will be some negative sideeffect for continue usage of this repo diff --git a/hydrodatasource/reader/floodevent.py b/hydrodatasource/reader/floodevent.py index 7865591..f9ffe47 100644 --- a/hydrodatasource/reader/floodevent.py +++ b/hydrodatasource/reader/floodevent.py @@ -419,9 +419,31 @@ def parse_augmented_file_metadata(self, augmented_file_path: str) -> Dict: ) elif "Sample ID:" in line: metadata["sample_id"] = int(line.split("Sample ID:")[1].strip()) + elif "Score:" in line: + # 提取评分信息 + import re + score_match = re.search(r"#\s*Score:\s*(\d+)", line) + if score_match: + score = int(score_match.group(1)) + # 限制评分范围0-100 + metadata["Score"] = max(0, min(100, score)) else: break + # 处理缺失Score的情况：根据时间分界线判断 + if "Score" not in metadata: + if "augmented_start_time" in metadata: + start_year = int(metadata["augmented_start_time"][:4]) + if start_year <= 2025: + # 真实数据：默认100分 + metadata["Score"] = 100 + else: + # 2026+增强数据无评分：标记为None（后续跳过） + metadata["Score"] = None + else: + # 无法判断时间：默认100分 + metadata["Score"] = 100 + return metadata def get_warmup_period_data( @@ -526,7 +548,11 @@ def adjust_warmup_time_to_augmented_year( return df def concatenate_warmup_and_augmented_data( - self, warmup_df: pd.DataFrame, augmented_file_path: str + self, + warmup_df: pd.DataFrame, + augmented_file_path: str, + use_score_as_weight: bool = False, # 新增参数 + score_threshold: int = 60, # 新增参数 ) -> pd.DataFrame: """ 拼接预热期数据和增强场次数据 @@ -534,7 +560,7 @@ def concatenate_warmup_and_augmented_data( Parameters ---------- warmup_df : pd.DataFrame - 预热期数据 + 预热期数据（仅在增强CSV未包含预热期时使用） augmented_file_path : str 增强文件路径 @@ -548,6 +574,49 @@ def concatenate_warmup_and_augmented_data( # 将时间列保持为字符串格式，避免超出pandas时间戳范围限制 aug_df["time"] = aug_df["time"].astype(str) + # 解析元数据获取event_id + metadata = self.parse_augmented_file_metadata(augmented_file_path) + + # 检查增强CSV是否已包含flood_event列（表示已包含预热期） + if "flood_event" in aug_df.columns: + # CSV文件已包含预热期和洪水期数据，无需拼接外部warmup + + # 解析元数据获取评分 + score = metadata.get("Score", None) + + # 根据模式处理 flood_event 列 + if use_score_as_weight and score is not None: + # 评分加权模式：将评分存储到 flood_event 列 + score = int(score) + + def apply_score_weighting(flood_event_value): + if flood_event_value == 0: + # 非洪水保持为 0 + return 0 + elif flood_event_value == 1: + # 洪水事件：根据评分决定 + if score < score_threshold: + return 1 # 低质量：标记为 1 + else: + return score # 高质量：使用实际评分 + else: + return flood_event_value # 其他值保持不变 + + aug_df["flood_event"] = aug_df["flood_event"].apply(apply_score_weighting) + print(f" 📊 Applied score weighting: score={score}, threshold={score_threshold}") + + # 添加 event_id 列 + if "augmented_start_time" in metadata and "augmented_end_time" in metadata: + event_id = f"event_{metadata['augmented_start_time']}_{metadata['augmented_end_time']}" + aug_df["event_id"] = aug_df["flood_event"].apply( + lambda x: event_id if x >= 1 else "" # 修改：>= 1 包含所有洪水事件 + ) + else: + aug_df["event_id"] = "" + + return aug_df + + # 旧版CSV文件不包含flood_event列，需要拼接外部warmup # 获取增强数据的起始时间，用于调整预热期数据的年份 if not aug_df.empty: # 从字符串格式的时间中提取年份 @@ -564,7 +633,29 @@ def concatenate_warmup_and_augmented_data( # 为预热期数据和增强数据添加标记列 adjusted_warmup_df["flood_event"] = 0 # 预热期数据标记为0 - aug_df["flood_event"] = 1 # 洪水期数据标记为1 + + # 根据模式设置洪水期的 flood_event 值 + score = metadata.get("Score", None) + + if use_score_as_weight and score is not None: + # 评分加权模式 + score = int(score) + if score < score_threshold: + aug_df["flood_event"] = 1 # 低质量：标记为 1 + else: + aug_df["flood_event"] = score # 高质量：使用实际评分 + print(f" 📊 Applied score weighting: score={score}, threshold={score_threshold}") + else: + # 阈值过滤模式 + aug_df["flood_event"] = 1 # 洪水期数据统一标记为1 + + # 添加event_id列（用于后续评分匹配） + if "augmented_start_time" in metadata and "augmented_end_time" in metadata: + event_id = f"event_{metadata['augmented_start_time']}_{metadata['augmented_end_time']}" + aug_df["event_id"] = event_id + else: + aug_df["event_id"] = "" + adjusted_warmup_df["event_id"] = "" # 预热期无event_id # 拼接数据并按字符串格式的时间排序 combined_df = pd.concat([adjusted_warmup_df, aug_df], ignore_index=True) @@ -642,47 +733,48 @@ def create_xarray_dataset_from_augdf( """ # The gen_discharge is the generated discharge by the data augmentation method # 创建数据集字典 + # 注意：维度顺序必须为 ["basin", "time"] 以与hydrodatasource标准格式兼容 data_vars = {} # 添加降雨数据 if "rain" in df.columns: data_vars["rain"] = ( - ["time", "basin"], - df[["rain"]].values.reshape(-1, 1), + ["basin", "time"], # 修正：维度顺序改为 basin 在前 + df[["rain"]].values.reshape(1, -1), # 修正：形状改为 (1, n_time) ) # 添加生成的流量数据 if "inflow" in df.columns: data_vars["inflow"] = ( - ["time", "basin"], - df[["inflow"]].values.reshape(-1, 1), + ["basin", "time"], + df[["inflow"]].values.reshape(1, -1), ) if "gen_discharge" in df.columns: data_vars["inflow"] = ( - ["time", "basin"], - df[["gen_discharge"]].values.reshape(-1, 1), + ["basin", "time"], + df[["gen_discharge"]].values.reshape(1, -1), ) # 添加观测流量数据 # if "obs_discharge" in df.columns: # data_vars["obs_discharge"] = ( - # ["time", "basin"], - # df[["obs_discharge"]].values.reshape(-1, 1), + # ["basin", "time"], + # df[["obs_discharge"]].values.reshape(1, -1), # ) # 洪水期标记 if "flood_event" in df.columns: data_vars["flood_event"] = ( - ["time", "basin"], - df[["flood_event"]].values.reshape(-1, 1), + ["basin", "time"], + df[["flood_event"]].values.reshape(1, -1), ) # 添加蒸散发数据 if "ES" in df.columns: data_vars["ES"] = ( - ["time", "basin"], - df[["ES"]].values.reshape(-1, 1), + ["basin", "time"], + df[["ES"]].values.reshape(1, -1), ) # 创建数据集 @@ -1044,6 +1136,8 @@ def process_augmented_files_by_discovery( latest_only: bool = True, warmup_hours: int = 240, time_unit: str = "3h", + score_threshold: int = 60, + use_score_as_weight:bool = False, ) -> Optional[str]: """ Process augmented data files based on file discovery. @@ -1066,6 +1160,16 @@ def process_augmented_files_by_discovery( Number of warmup hours. time_unit : str, optional Time unit. + score_threshold : int, optional + Score threshold (default: 60). Behavior depends on use_score_as_weight: + - If use_score_as_weight=False: Only events with Score >= score_threshold are included (filtering mode) + - If use_score_as_weight=True: Events with Score < score_threshold get flood_event=1, + events with Score >= score_threshold get flood_event=score (weighting mode) + use_score_as_weight : bool, optional + If True, use score-based weighting mode where flood_event stores actual scores (1-100) + instead of binary values (0/1). Default is False (threshold filtering mode). + - False (filtering): flood_event=0/1, only score>=threshold included + - True (weighting): flood_event=-1/0 (non-flood), 1 (low-quality), or score (high-quality) Returns ------- @@ -1089,17 +1193,17 @@ def process_augmented_files_by_discovery( print("❌ No augmented data files were found") return None print(f"🔄 Prepare to process {len (discovered_files)} augmented data files:") - # for file_info in discovered_files: - # modified_by = file_info.get("modified_by", "unknown") - # print(f" - {file_info['filename']} (修改者: {modified_by})") + # 创建文件路径列表，用于现有的处理方法 file_paths = [file_info["file_path"] for file_info in discovered_files] - # 调用现有的处理方法，但使用文件路径而不是编号 + # 调用现有的处理方法，传递 use_score_as_weight 参数 return self._process_augmented_files_by_paths( station_ids=station_ids, file_paths=file_paths, warmup_hours=warmup_hours, time_unit=time_unit, + score_threshold=score_threshold, + use_score_as_weight=use_score_as_weight, # 新增 ) def _process_augmented_files_by_paths( @@ -1108,25 +1212,31 @@ def _process_augmented_files_by_paths( file_paths: List[str], warmup_hours: int = 240, time_unit: str = "3h", + score_threshold: int = 60, + use_score_as_weight: bool = False, # 新增参数 ) -> Optional[str]: """ - Process augmented data files based on file paths. + 处理增强数据文件基于文件路径（结构化数组版）- 解决字典保存问题 Parameters ---------- station_ids : Union[str, List[str]] - Station ID or list of station IDs. + 站点ID或站点ID列表 file_paths : List[str] - List of file paths to process. + 要处理的文件路径列表 warmup_hours : int, optional - Number of warmup hours. + 预热小时数，默认240 time_unit : str, optional - Time unit. + 时间单位，默认"3h" + score_threshold : int, optional + 分数阈值，默认60。根据 use_score_as_weight 有不同行为 + use_score_as_weight : bool, optional + 是否使用评分加权模式，默认False（阈值过滤模式） Returns ------- Optional[str] - Path to the cache file, or None if processing fails. + 缓存文件路径，处理失败返回None """ # 兼容单个站点的情况 if isinstance(station_ids, str): @@ -1137,73 +1247,105 @@ def _process_augmented_files_by_paths( # 对每个站点处理增强数据 for station_id in station_ids: - print(f"🔄 Processing : {station_id}") + print(f"🔄🔄 Processing station: {station_id}") station_timeseries = [] + total_files = len(file_paths) # 总文件数 + processed_files = 0 # 成功处理的文件数 for file_path in file_paths: try: - print(f" 🔄 Processing file: {os.path.basename(file_path)}") + print(f" 🔄🔄 Processing file: {os.path.basename(file_path)}") # 解析增强文件的元信息 metadata = self.parse_augmented_file_metadata(file_path) if not metadata: - print( - f" ⚠️ Skip file {file_path}: unable to parse metadata" - ) + print(f" ⚠⚠⚠️ Skip file {file_path}: unable to parse metadata") continue - # 获取预热期数据 - warmup_df = self.get_warmup_period_data( - original_start_time=metadata.get("original_start_time"), - original_end_time=metadata.get("original_end_time"), - station_id=station_id, - warmup_hours=warmup_hours, - ) + # 跳过2026+无评分的文件 + if "Score" in metadata and metadata["Score"] is None: + print(f" ⚠⚠⚠️ Skip unscored augmented data file: {os.path.basename(file_path)}") + continue - if warmup_df is None: - print( - f" ⚠️ Skip file {file_path}: unable to get warmup data" - ) + # 根据模式处理评分 + if "Score" in metadata: + score = int(metadata["Score"]) + if use_score_as_weight: + # 评分加权模式：所有数据都接受，稍后在 flood_event 中存储评分 + print(f" ✅ Accept file {os.path.basename(file_path)}: Score {score} (weighting mode)") + else: + # 阈值过滤模式：只接受 >= threshold 的数据 + if score < score_threshold: + print(f" ⚠⚠⚠️ Skip file {os.path.basename(file_path)}: Score {score} < threshold {score_threshold}") + continue + else: + print(f" ✅ Accept file {os.path.basename(file_path)}: Score {score} >= threshold {score_threshold}") + else: + # 真实数据（无Score字段）默认通过 + print(f" ✅ Accept file {os.path.basename(file_path)}: Real data (no score)") + + # 检查CSV文件是否已包含flood_event列 + try: + sample_df = pd.read_csv(file_path, comment="#", nrows=1) + has_flood_event = "flood_event" in sample_df.columns + except Exception as e: + print(f" ⚠⚠⚠️ Skip file {file_path}: unable to read CSV header - {str(e)}") continue + # 仅当CSV不包含flood_event列时，才获取外部预热期数据 + if has_flood_event: + warmup_df = pd.DataFrame() + else: + # 获取预热期数据 + warmup_df = self.get_warmup_period_data( + original_start_time=metadata.get("original_start_time", ""), + original_end_time=metadata.get("original_end_time", ""), + station_id=station_id, + warmup_hours=warmup_hours, + ) + + if warmup_df is None: + print(f" ⚠⚠⚠️ Skip file {file_path}: unable to get warmup data") + continue + # 调整预热期时间并拼接增强数据 timeseries_df = self.concatenate_warmup_and_augmented_data( - warmup_df, file_path + warmup_df, + file_path, + use_score_as_weight=use_score_as_weight, # 新增 + score_threshold=score_threshold, # 新增 ) if timeseries_df is not None and len(timeseries_df) > 0: station_timeseries.append(timeseries_df) - # print(f" ✅ 成功处理: {len(timeseries_df)} 条记录") + processed_files += 1 # 成功处理文件计数 else: - print( - f" ⚠️ Skip file {file_path}: The processed data is empty" - ) + print(f" ⚠⚠⚠️ Skip file {file_path}: The processed data is empty") + except Exception as e: - print(f" ❌ Processing file failed {file_path}: {str(e)}") + print(f" ❌❌ Processing file failed {file_path}: {str(e)}") continue # 如果该站点有数据，则合并并转换为xarray Dataset if station_timeseries: - print(f" 🔄 Merge time series data of {station_id}...") + print(f"\n 🔄🔄 Merge time series data of {station_id}...") + print(f" 📊 Successfully merged {processed_files}/{total_files} augmented files") combined_df = pd.concat(station_timeseries, ignore_index=True) combined_df = combined_df.sort_values("time").reset_index(drop=True) + # 重命名列 gen_discharge -> inflow combined_df = self.rename_dataframe_columns(combined_df) # 转换为xarray Dataset - station_ds = self.create_xarray_dataset_from_augdf( - combined_df, station_id, time_unit - ) + station_ds = self.create_xarray_dataset_from_augdf(combined_df, station_id, time_unit) all_datasets.append(station_ds) processed_count += 1 - print( - f" ✅ {station_id} processing completed: {len (combined_df)} records" - ) + print(f" ✅ {station_id} processing completed: {len(combined_df)} records\n") else: - print(f" ⚠️ {station_id} No data available") + print(f" ⚠⚠⚠️ {station_id} No data available") if not all_datasets: - print("❌ Failed to process data from any basin successfully") + print("❌❌ Failed to process data from any basin successfully") return None print(f"✅ Successfully processed {processed_count} basins") @@ -1212,14 +1354,25 @@ def _process_augmented_files_by_paths( if len(all_datasets) == 1: final_ds = all_datasets[0] else: - print("🔄 Merge datasets from multiple basins...") - final_ds = xr.concat(all_datasets, dim="gage_id") + print("🔄🔄 Merge datasets from multiple basins...") + final_ds = xr.concat(all_datasets, dim="basin") # 修正：使用basin而非gage_id + + # 添加数据集属性 + final_ds.attrs.update({ + 'processing_timestamp': datetime.now().isoformat(), + 'data_source': 'augmented_flood_events', + 'use_score_as_weight': int(use_score_as_weight), # 转换 boolean 为 int (NetCDF 不支持 boolean 类型) + }) # 保存到缓存 - print("🔄 Save augmented data to cache...") - cache_file_path = self.save_augmented_timeseries_to_cache( - final_ds, station_ids, time_unit - ) + print("🔄🔄 Save augmented data to cache...") + cache_file_path = self.save_augmented_timeseries_to_cache(final_ds, station_ids, time_unit) + + if cache_file_path: + print(f"✅ Successfully saved to: {cache_file_path}") + else: + print("❌ Failed to save cache file") + return cache_file_path def get_user_contributions_summary(self, augmented_files_dir: str) -> pd.DataFrame: From 58fcf3a1672a6dabc3695183205c50e4ed6b39e6 Mon Sep 17 00:00:00 2001 From: zyb222333 <2360828996@qq.com> Date: Thu, 2 Apr 2026 21:37:04 +0800 Subject: [PATCH 11/34] dataaug --- hydrodatasource/configs/config.py | 6 ++++-- hydrodatasource/reader/floodevent.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/hydrodatasource/configs/config.py b/hydrodatasource/configs/config.py index 768fbb4..e520321 100644 --- a/hydrodatasource/configs/config.py +++ b/hydrodatasource/configs/config.py @@ -49,9 +49,10 @@ def read_setting(setting_path): ) # Define the expected structure + # Note: cache is now optional expected_structure = { "minio": ["server_url", "client_endpoint", "access_key", "secret"], - "local_data_path": ["root", "datasets-origin", "datasets-interim", "cache"], + "local_data_path": ["root", "datasets-origin", "datasets-interim"], "postgres": ["server_url", "port", "username", "password", "database"], } @@ -77,7 +78,8 @@ def read_setting(setting_path): try: SETTING = read_setting(SETTING_FILE) LOCAL_DATA_PATH = SETTING["local_data_path"]["root"] - CACHE_DIR = SETTING["local_data_path"]["cache"] + # cache is optional, use default if not provided + CACHE_DIR = SETTING["local_data_path"].get("cache", Path.home().joinpath("hydrodatasource_data", ".cache")) except (ValueError, FileNotFoundError) as e: LOCAL_DATA_PATH = Path.home().joinpath("hydrodatasource_data") CACHE_DIR = Path.home().joinpath("hydrodatasource_data", ".cache") diff --git a/hydrodatasource/reader/floodevent.py b/hydrodatasource/reader/floodevent.py index 7865591..d6c4967 100644 --- a/hydrodatasource/reader/floodevent.py +++ b/hydrodatasource/reader/floodevent.py @@ -92,6 +92,34 @@ def __init__( **kwargs, ) + # 验证数据目录是否存在 + if not os.path.exists(self.data_source_dir): + raise FileNotFoundError( + f"❌ 数据集目录不存在: {self.data_source_dir}\n" + f"\n配置信息:" + f"\n data_path = {data_path}" + f"\n dataset_name = {dataset_name}" + f"\n 拼接后路径 = {self.data_source_dir}" + f"\n\n请检查:" + f"\n 1. data_path 是否指向数据集的父目录" + f"\n 2. dataset_name 拼写是否正确" + f"\n 3. 数据集目录是否实际存在" + ) + + # 验证必需的子目录 + required_subdirs = ["attributes", "timeseries"] + missing_dirs = [] + for subdir in required_subdirs: + subdir_path = os.path.join(self.data_source_dir, subdir) + if not os.path.exists(subdir_path): + missing_dirs.append(subdir) + + if missing_dirs: + raise FileNotFoundError( + f"❌ 数据集目录结构不完整: {self.data_source_dir}\n" + f"缺少以下子目录: {', '.join(missing_dirs)}" + ) + def get_constants(self): """ Get the constant values used by this datasource. From 8482bdf02d66ece40c0f2a9937f0353b61229ee3 Mon Sep 17 00:00:00 2001 From: owen Date: Thu, 18 Jun 2026 09:22:20 +0800 Subject: [PATCH 12/34] feat: add data_resolver.py with READER_ALIASES for all 11 hydrodatasource reader classes - Register all HydroData subclasses: floodevent, selfmade, longterm, forecast, station, tghydro, gages, grdc, rainfall, crd, rsvrinflow - Implement resolve_data_path() following same contract as hydromodel ADR 0001 - Merge hydrodataset's READER_ALIASES (37 entries) for a combined 44+ registry - Re-export DatasetResolutionError from hydrodataset - Add comprehensive tests (19 tests covering aliases, errors, local/cloud resolution) Co-Authored-By: Claude --- hydrodatasource/configs/data_resolver.py | 293 +++++++++++++++++++++++ tests/test_data_resolver.py | 168 +++++++++++++ 2 files changed, 461 insertions(+) create mode 100644 hydrodatasource/configs/data_resolver.py create mode 100644 tests/test_data_resolver.py diff --git a/hydrodatasource/configs/data_resolver.py b/hydrodatasource/configs/data_resolver.py new file mode 100644 index 0000000..7c02267 --- /dev/null +++ b/hydrodatasource/configs/data_resolver.py @@ -0,0 +1,293 @@ +""" +Reader alias registry and data path resolution for hydrodatasource. + +Defines READER_ALIASES for all hydrodatasource reader classes and provides +resolve_data_path() following the same contract as hydromodel's ADR 0001. + +Reuses hydrodataset's storage configuration parsing (settings.py) and +validation logic, so all three repos produce the same resolved URIs. + +Usage: + from hydrodatasource.configs.data_resolver import ( + READER_ALIASES, + resolve_data_path, + DatasetResolutionError, + ) + + uri = resolve_data_path("songliao_event", project_root=".") + ds = FloodEventDatasource(data_path=uri, dataset_name="myevents") +""" + +from __future__ import annotations + +from pathlib import Path, PurePosixPath, PureWindowsPath +from typing import Any, Dict, Optional + +import yaml + +from hydrodataset.configs.data_resolver import ( + DatasetResolutionError, + READER_ALIASES as _HD_READER_ALIASES, +) +from hydrodataset.configs.settings import get_local_root, get_storage_config + +# ── hydrodatasource-specific reader aliases ────────────────────────────── + +_HDS_READER_ALIASES: Dict[str, Dict[str, str]] = { + "floodevent": { + "module": "hydrodatasource.reader.floodevent", + "class": "FloodEventDatasource", + "category": "hydrodatasource", + }, + "selfmade": { + "module": "hydrodatasource.reader.data_source", + "class": "SelfMadeHydroDataset", + "category": "hydrodatasource", + }, + "longterm": { + "module": "hydrodatasource.reader.data_source", + "class": "LongTermDataset", + "category": "hydrodatasource", + }, + "forecast": { + "module": "hydrodatasource.reader.data_source", + "class": "SelfMadeForecastDataset", + "category": "hydrodatasource", + }, + "station": { + "module": "hydrodatasource.reader.data_source", + "class": "StationHydroDataset", + "category": "hydrodatasource", + }, + "tghydro": { + "module": "hydrodatasource.reader.data_source", + "class": "TgHydroDatasource", + "category": "hydrodatasource", + }, + "gages": { + "module": "hydrodatasource.reader.gages", + "class": "Gages", + "category": "hydrodatasource", + }, + "grdc": { + "module": "hydrodatasource.reader.grdc", + "class": "Grdc", + "category": "hydrodatasource", + }, + "rainfall": { + "module": "hydrodatasource.reader.rainfall_reader", + "class": "RainfallReader", + "category": "hydrodatasource", + }, + "crd": { + "module": "hydrodatasource.reader.reservoir_datasets", + "class": "Crd", + "category": "hydrodatasource", + }, + "rsvrinflow": { + "module": "hydrodatasource.reader.rsvr_inflow_reader", + "class": "RsvrInflowReader", + "category": "hydrodatasource", + }, +} + +# Merged: hydrodataset's aliases first, then hydrodatasource overrides +READER_ALIASES: Dict[str, Dict[str, str]] = { + **_HD_READER_ALIASES, + **_HDS_READER_ALIASES, +} + +FORBIDDEN_PATH_PATTERNS = {"://", ".."} + + +def _load_yaml(path: Path) -> Dict[str, Any]: + """Load a YAML file, returning {} if it does not exist.""" + if not path.exists(): + return {} + with path.open("r", encoding="utf-8") as f: + loaded = yaml.safe_load(f) + return loaded or {} + + +def _load_registry(project_root: Path) -> Dict[str, Dict[str, Any]]: + """Load dataset registry, checking hydrodatasource first then hydrodataset. + + Looks in these locations: + 1. {project_root}/configs/datasets.yml (project-level, takes precedence) + 2. hydrodatasource package's own configs/datasets.yml + 3. hydrodataset/configs/datasets.yml (package fallback) + + Parameters + ---------- + project_root : Path + Root of the calling project. + + Returns + ------- + dict + Dataset registry mapping dataset_id -> {'reader': ..., 'path': ...} + + Raises + ------ + DatasetResolutionError + If no registry file is found. + """ + # Project-level config (takes precedence) + project_registry = project_root / "configs" / "datasets.yml" + # hydrodatasource package-internal fallback + package_registry = ( + Path(__file__).resolve().parent.parent.parent / "configs" / "datasets.yml" + ) + # hydrodataset's own datasets.yml + hd_registry = ( + Path(__file__).resolve().parent.parent.parent.parent + / "hydrodataset" + / "configs" + / "datasets.yml" + ) + + registry_path = None + if project_registry.exists(): + registry_path = project_registry + elif package_registry.exists(): + registry_path = package_registry + elif hd_registry.exists(): + registry_path = hd_registry + + if registry_path is None: + raise DatasetResolutionError( + "Dataset registry not found. Tried:\n" + f" - {project_registry}\n" + f" - {package_registry}\n" + f" - {hd_registry}\n" + "Create configs/datasets.yml in your project." + ) + + data = _load_yaml(registry_path) + datasets = data.get("datasets") + if not isinstance(datasets, dict): + raise DatasetResolutionError( + f"Dataset registry in {registry_path} must have a 'datasets' mapping." + ) + return datasets + + +def _validate_relative_path(path_value: str, dataset_id: str) -> None: + """Ensure path is a safe relative path (no URI schemes, no .., no absolute).""" + if not isinstance(path_value, str): + raise DatasetResolutionError( + f"Dataset '{dataset_id}' path must be a string, " + f"got {type(path_value)}" + ) + for forbidden in FORBIDDEN_PATH_PATTERNS: + if forbidden in path_value: + raise DatasetResolutionError( + f"Dataset '{dataset_id}' path must be relative, " + f"not contain '{forbidden}'" + ) + windows_path = PureWindowsPath(path_value) + posix_path = PurePosixPath(path_value) + if windows_path.is_absolute() or posix_path.is_absolute(): + raise DatasetResolutionError( + f"Dataset '{dataset_id}' path must be relative, " + f"got absolute: '{path_value}'" + ) + + +def resolve_data_path( + dataset_id: str, + *, + source: str = "local", + project_root: Optional[str] = None, +) -> str: + """Resolve a dataset id to an absolute data path (URI). + + Combines the dataset registry entry with storage configuration + to produce a single absolute path. Follows the same contract as + hydromodel's resolve_data_cfgs and hydrodataset's resolve_data_path. + + Parameters + ---------- + dataset_id : str + Dataset identifier from the registry (e.g. 'camels_us', 'songliao_event'). + source : str + Storage backend: 'local' or 'cloud'. + project_root : str, optional + Root of the calling project (for finding configs/datasets.yml). + Defaults to current working directory. + + Returns + ------- + str + Absolute URI pointing to the dataset's data directory. + + Raises + ------ + DatasetResolutionError + If any resolution step fails. + """ + if source not in {"local", "cloud"}: + raise DatasetResolutionError( + f"source must be 'local' or 'cloud', got '{source}'" + ) + + root = Path(project_root) if project_root else Path.cwd() + registry = _load_registry(root) + + if dataset_id not in registry: + known = ", ".join(sorted(registry)) + raise DatasetResolutionError( + f"Unknown dataset id '{dataset_id}'. " + f"Known datasets: {known}" + ) + + dataset_spec = registry[dataset_id] + reader = dataset_spec.get("reader") + if not reader: + raise DatasetResolutionError( + f"Dataset '{dataset_id}' must define 'reader'" + ) + if reader not in READER_ALIASES: + raise DatasetResolutionError( + f"Unknown reader alias '{reader}' for dataset '{dataset_id}'" + ) + + relative_path = dataset_spec.get("path") + if not relative_path: + raise DatasetResolutionError( + f"Dataset '{dataset_id}' must define 'path'" + ) + _validate_relative_path(relative_path, dataset_id) + + if source == "local": + root_dir = get_local_root() + if root_dir is None: + raise DatasetResolutionError( + "storage.local.root is not configured. " + "Set it in ~/hydro_setting.yml" + ) + if not root_dir.exists(): + raise DatasetResolutionError( + f"storage.local.root does not exist: {root_dir}" + ) + resolved = root_dir / relative_path + if not resolved.exists(): + raise DatasetResolutionError( + f"Resolved dataset path does not exist: {resolved}" + ) + return str(resolved) + + # cloud source + storage = get_storage_config() + s3 = storage.get("s3") + if not isinstance(s3, dict): + raise DatasetResolutionError( + "storage.s3 is required for cloud source" + ) + bucket = s3.get("bucket") + if not bucket: + raise DatasetResolutionError("storage.s3.bucket is required") + prefix = str(s3.get("prefix") or "").strip("/") + rel = relative_path.replace("\\", "/").strip("/") + path = f"{prefix}/{rel}" if prefix else rel + return f"s3://{bucket}/{path}" diff --git a/tests/test_data_resolver.py b/tests/test_data_resolver.py new file mode 100644 index 0000000..3ade2c0 --- /dev/null +++ b/tests/test_data_resolver.py @@ -0,0 +1,168 @@ +"""Tests for hydrodatasource.configs.data_resolver.""" + +import os +import tempfile +from pathlib import Path +from unittest.mock import patch + +import pytest +import yaml + + +class TestReaderAliases: + """Verify READER_ALIASES covers all hydrodatasource reader classes.""" + + def test_reader_aliases_not_empty(self): + from hydrodatasource.configs.data_resolver import READER_ALIASES + + assert isinstance(READER_ALIASES, dict) + assert len(READER_ALIASES) >= 11 + + def test_floodevent_alias_registered(self): + from hydrodatasource.configs.data_resolver import READER_ALIASES + + assert "floodevent" in READER_ALIASES + spec = READER_ALIASES["floodevent"] + assert spec["module"] == "hydrodatasource.reader.floodevent" + assert spec["class"] == "FloodEventDatasource" + assert spec["category"] == "hydrodatasource" + + def test_selfmade_alias_registered(self): + from hydrodatasource.configs.data_resolver import READER_ALIASES + + assert "selfmade" in READER_ALIASES + spec = READER_ALIASES["selfmade"] + assert spec["class"] == "SelfMadeHydroDataset" + + @pytest.mark.parametrize( + "alias", + [ + "floodevent", + "selfmade", + "longterm", + "forecast", + "station", + "tghydro", + "gages", + "grdc", + "rainfall", + "crd", + "rsvrinflow", + ], + ) + def test_all_aliases_have_required_fields(self, alias): + from hydrodatasource.configs.data_resolver import READER_ALIASES + + spec = READER_ALIASES[alias] + assert "module" in spec, f"{alias} missing 'module'" + assert "class" in spec, f"{alias} missing 'class'" + assert "category" in spec, f"{alias} missing 'category'" + assert spec["category"] == "hydrodatasource" + + +class TestDatasetResolutionError: + """Verify DatasetResolutionError is importable.""" + + def test_error_importable(self): + from hydrodatasource.configs.data_resolver import DatasetResolutionError + + err = DatasetResolutionError("test message") + assert isinstance(err, ValueError) + assert str(err) == "test message" + + +class TestResolveDataPath: + """Verify resolve_data_path with mocked storage and registry.""" + + @pytest.fixture + def temp_root(self, tmp_path): + """Create a temp directory that serves as storage.local.root.""" + data_dir = tmp_path / "data" + data_dir.mkdir(parents=True) + dataset_dir = data_dir / "projects" / "songliao" / "event" + dataset_dir.mkdir(parents=True) + return tmp_path + + @pytest.fixture + def datasets_yml(self, tmp_path): + """Create a minimal configs/datasets.yml.""" + configs_dir = tmp_path / "configs" + configs_dir.mkdir(parents=True) + yml_path = configs_dir / "datasets.yml" + yml_path.write_text( + yaml.dump( + { + "datasets": { + "songliao_event": { + "reader": "floodevent", + "path": "projects/songliao/event", + } + } + } + ), + encoding="utf-8", + ) + return tmp_path + + def test_resolve_local_path(self, temp_root, datasets_yml, monkeypatch): + """Resolve a dataset to an absolute local path.""" + from hydrodatasource.configs.data_resolver import resolve_data_path + + monkeypatch.setattr( + "hydrodatasource.configs.data_resolver.get_local_root", + lambda: temp_root / "data", + ) + + uri = resolve_data_path( + "songliao_event", source="local", project_root=str(datasets_yml) + ) + expected = str(temp_root / "data" / "projects" / "songliao" / "event") + assert uri == expected + + def test_resolve_unknown_dataset_raises(self, datasets_yml, monkeypatch, tmp_path): + """Resolving an unknown dataset id should raise.""" + from hydrodatasource.configs.data_resolver import ( + resolve_data_path, + DatasetResolutionError, + ) + + monkeypatch.setattr( + "hydrodatasource.configs.data_resolver.get_local_root", + lambda: tmp_path / "data", + ) + + with pytest.raises(DatasetResolutionError): + resolve_data_path( + "nonexistent_dataset", source="local", project_root=str(datasets_yml) + ) + + def test_invalid_source_raises(self, datasets_yml): + """Invalid source should raise.""" + from hydrodatasource.configs.data_resolver import ( + resolve_data_path, + DatasetResolutionError, + ) + + with pytest.raises(DatasetResolutionError): + resolve_data_path( + "songliao_event", source="ftp", project_root=str(datasets_yml) + ) + + def test_resolve_cloud_s3_uri(self, datasets_yml, monkeypatch): + """Resolve a dataset to an S3 URI for cloud source.""" + from hydrodatasource.configs.data_resolver import resolve_data_path + + monkeypatch.setattr( + "hydrodatasource.configs.data_resolver.get_storage_config", + lambda: { + "s3": { + "bucket": "hydro-data", + "prefix": "hydromodel", + } + }, + ) + + uri = resolve_data_path( + "songliao_event", source="cloud", project_root=str(datasets_yml) + ) + assert uri == "s3://hydro-data/hydromodel/projects/songliao/event" From 8614e9b05269f78409f635370237e55d70e3516b Mon Sep 17 00:00:00 2001 From: owen Date: Thu, 18 Jun 2026 09:31:36 +0800 Subject: [PATCH 13/34] chore: add local hydrodataset source and unified data interface spec - Add tool.uv.sources pointing to local hydrodataset for development - Include spec document for unified data interface alignment Co-Authored-By: Claude --- docs/spec-unified-data-interface.md | 188 ++++++++++++++++++++++++++++ pyproject.toml | 3 + uv.lock | 82 ++++++++++-- 3 files changed, 264 insertions(+), 9 deletions(-) create mode 100644 docs/spec-unified-data-interface.md diff --git a/docs/spec-unified-data-interface.md b/docs/spec-unified-data-interface.md new file mode 100644 index 0000000..78f0822 --- /dev/null +++ b/docs/spec-unified-data-interface.md @@ -0,0 +1,188 @@ +# Spec: Unified Data Interface Alignment (hydrodatasource) + +## Status + +Approved — ready for implementation. + +## Objective + +Align `hydrodatasource` with `hydromodel`'s [ADR 0001: Unified Data Path Resolution](../hydromodel/docs/adr/0001-unified-data-path-resolution.md) +and `hydrodataset`'s configuration system, so all three repositories share one deterministic +contract for resolving dataset identity → storage location → runtime URI. + +**Key principles:** +- hydrodatasource should reuse hydrodataset's config parsing code — NOT reinvent wheels +- hydrodatasource contributes its own **reader aliases** and **dataset registry**, which hydromodel consumes alongside hydrodataset's +- Database and legacy MinIO configuration must be **removed** — storage is unified under `storage.local` / `storage.s3` (cloud) +- All hydrodatasource reader classes must be registered as reader aliases + +## Resolved Questions + +| Question | Answer | +|----------|--------| +| Explicit hydrodataset dependency? | **Yes** — add to pyproject.toml since we import its settings.py and data_resolver.py directly | +| Old minio config? | **Remove** — superseded by `storage.s3` (cloud) | +| Old postgres config? | **Remove** — database access belongs in a real-time service repo, not coupled here | +| hydromodel dataset_dict.py? | **Deprecate/remove** — superseded by merged READER_ALIASES from both packages | +| Which readers to register? | **All of them** — every HydroData subclass gets a reader alias | + +## Current State + +### What exists today + +| Repo | data_resolver | datasets.yml | READER_ALIASES | settings.py | +|------|--------------|-------------|----------------|-------------| +| hydromodel | `hydromodel/configs/data_resolver.py` | `configs/datasets.yml` (2 datasets) | 4 entries (hardcoded) | built into data_resolver | +| hydrodataset | `hydrodataset/configs/data_resolver.py` | `configs/datasets.yml` (37 datasets) | 37 entries | `hydrodataset/configs/settings.py` | +| hydrodatasource | **NONE** | **NONE** | **NONE** | Old-style `configs/config.py` with import-time side effects | + +### The gap + +``` +OLD (current hydrodatasource): + ~/hydro_setting.yml (local_data_path.root, minio.*, postgres.*) + → configs/config.py (import-time side effects: SETTING, CACHE_DIR, FS, PS) + → reader classes (data_path + dataset_name) + +NEW (target): + ~/hydro_setting.yml (storage.local.root, storage.s3.* — NO minio, NO postgres) + → hydrodataset.configs.settings (pure functions) + + hydrodatasource.configs.data_resolver (READER_ALIASES + resolve_data_path) + + configs/datasets.yml (dataset id → reader + relative path) + → reader classes (accept resolved URI) +``` + +### Problems to fix + +1. Import-time side effects in `configs/config.py` +2. Old config format (`local_data_path.root` vs new `storage.local.root`) +3. No dataset registry (no `datasets.yml`) +4. No reader aliases for hydrodatasource's readers +5. Path-based constructors instead of URI-accepting ones +6. **minio config block — REMOVE** (cloud access via `storage.s3` only) +7. **postgres config block — REMOVE** (database decoupled from this repo) + +## Tech Stack + +- Python 3.10+ +- PyYAML for config parsing +- pathlib for path operations +- **hydrodataset** as explicit dependency in pyproject.toml + +## Commands + +```bash +uv sync --dev +uv run pytest tests/ -v +uv run black hydrodatasource/ +uv run flake8 hydrodatasource/ +uv build +``` + +## Design Decisions + +### Decision 1: Reuse hydrodataset's settings.py directly + +hydrodatasource will import `get_local_root`, `get_cache_dir`, `load_settings` from +`hydrodataset.configs.settings`. + +### Decision 2: hydrodatasource gets its own data_resolver.py + +`hydrodatasource/configs/data_resolver.py` with: +- `READER_ALIASES` for ALL hydrodatasource reader classes +- `resolve_data_path()` delegating to hydrodataset's logic +- `DatasetResolutionError` re-exported from hydrodataset + +### Decision 3: Two-stage migration for config.py + +**Stage A (this spec):** Add new functions, keep `CACHE_DIR`/`SETTING` globals via lazy compat layer. Remove minio/postgres globals. +**Stage B (future):** Deprecate and remove all import-time side effects. + +### Decision 4: Remove minio and postgres config blocks + +The `~/hydro_setting.yml` for hydrodatasource must NOT contain `minio.*` or `postgres.*`. +Cloud storage uses `storage.s3.*`. Database access is decoupled to a separate real-time service. + +### Decision 5: Register ALL reader classes + +All 11 HydroData subclasses get a reader alias. + +### Decision 6: READER_ALIASES format follows hydrodataset + +```python +READER_ALIASES: Dict[str, Dict[str, str]] = { + "alias_name": { + "module": "hydrodatasource.reader.floodevent", + "class": "FloodEventDatasource", + "category": "hydrodatasource", + }, +} +``` + +## Implementation Phases + +### Phase 1: Create data_resolver.py for hydrodatasource + +**File:** `hydrodatasource/configs/data_resolver.py` + +Register ALL reader classes: + +| Alias | Module | Class | +|-------|--------|-------| +| `floodevent` | hydrodatasource.reader.floodevent | FloodEventDatasource | +| `selfmade` | hydrodatasource.reader.data_source | SelfMadeHydroDataset | +| `longterm` | hydrodatasource.reader.data_source | LongTermDataset | +| `forecast` | hydrodatasource.reader.data_source | SelfMadeForecastDataset | +| `station` | hydrodatasource.reader.data_source | StationHydroDataset | +| `tghydro` | hydrodatasource.reader.data_source | TgHydroDatasource | +| `gages` | hydrodatasource.reader.gages | Gages | +| `grdc` | hydrodatasource.reader.grdc | Grdc | +| `rainfall` | hydrodatasource.reader.rainfall_reader | RainfallReader | +| `crd` | hydrodatasource.reader.reservoir_datasets | Crd | +| `rsvrinflow` | hydrodatasource.reader.rsvr_inflow_reader | RsvrInflowReader | + +Also implements `resolve_data_path(dataset_id, *, source, project_root)` and re-exports `DatasetResolutionError`. + +### Phase 2: Create top-level configs/datasets.yml + +**File:** `configs/datasets.yml` (repo root) + +### Phase 3: Refactor configs/config.py — additive + remove minio/postgres + +- Add lazy loading via `__getattr__` at module level +- Add `get_local_root()` → delegates to `hydrodataset.configs.settings` +- Add `get_cache_dir()` → same +- Keep `CACHE_DIR`, `SETTING`, `LOCAL_DATA_PATH` as deprecated compat +- **Remove:** `MINIO_PARAM`, `RO`, `S3`, `MC`, `FS` globals +- **Remove:** `POSTGRES_PARAM`, `PS`, `get_postgres_connection()` +- **Remove:** `read_setting()` validation of `minio` and `postgres` sections + +### Phase 4: Add URI-accepting constructors to reader classes + +Update `HydroData.__init__` and subclasses to accept `uri` parameter. + +### Phase 5: Update hydromodel data_resolver.py + +Import and merge READER_ALIASES from both hydrodataset and hydrodatasource. +Deprecate `dataset_dict.py`. + +### Phase 6: Add hydrodataset dependency to pyproject.toml + +## Success Criteria + +- [ ] `hydrodatasource/configs/data_resolver.py` with READER_ALIASES for ALL 11 reader classes +- [ ] `configs/datasets.yml` at repo root +- [ ] `resolve_data_path("songliao_event")` resolves correctly +- [ ] `from hydrodatasource.configs.config import CACHE_DIR` still works +- [ ] No `minio` or `postgres` config validation in config.py +- [ ] `SelfMadeHydroDataset(uri="/data/...")` works +- [ ] hydromodel can import and merge hydrodatasource's READER_ALIASES +- [ ] No import-time side effects in new code paths +- [ ] Same `~/hydro_setting.yml` (`storage.local.root`) works for all three repos +- [ ] hydrodataset is an explicit dependency in pyproject.toml + +## Boundaries + +- **Always do:** Use hydrodataset's settings.py; define reader aliases in Python code; validate paths per ADR 0001; use `storage.local.root` format +- **Ask first:** Removing old `config.py` globals entirely; changing reader class constructor signatures incompatibly +- **Never do:** Define Python module import paths in YAML; create datasets with absolute paths; allow `..` in dataset paths; commit secrets; add `minio.*` or `postgres.*` config validation back diff --git a/pyproject.toml b/pyproject.toml index 133d0eb..cdfe6bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -175,6 +175,9 @@ exclude = [ ".tox", ] +[tool.uv.sources] +hydrodataset = { path = "../hydrodataset", editable = true } + [tool.uv] # Override aqua-fetch's outdated dependency constraints # aqua-fetch 限制了 pandas<=2.1.4，但实际上新版本的 pandas 和 xarray 已经修复了兼容性问题 diff --git a/uv.lock b/uv.lock index e4197a0..31a31ab 100644 --- a/uv.lock +++ b/uv.lock @@ -258,15 +258,15 @@ wheels = [ [[package]] name = "aqua-fetch" -version = "1.0.0" +version = "1.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pandas" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f6/b9/aefde1dae1265a1db6bd0d9163a53e5823649cc11ed0a8b8eb82fbe3f3fd/aqua_fetch-1.0.0.tar.gz", hash = "sha256:92d1bf97d17786b2841dab6544161891d1c29bda224333b5261fef53634820cd", size = 255512, upload-time = "2025-08-24T10:16:53.866Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a9/7e/b4343a680a1f0b2356f6cf9371dded8141c0d8ea61ec7b41c6cdbedab3fe/aqua_fetch-1.0.1.tar.gz", hash = "sha256:7991a7fa2ab6592ff23dd0f8b9c2e2280b7aca20b01e4731bc3b6f974ded9578", size = 274826, upload-time = "2026-02-11T08:02:55.734Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/cb/b12598547cb67b195266d28e5c1e390e0c7f1d54bf5a8d9a0c3d954ac5f0/aqua_fetch-1.0.0-py3-none-any.whl", hash = "sha256:2c685b6463453aef2cb197079f9b98a76d1fce5a045731ada697d7b3d44289d6", size = 269490, upload-time = "2025-08-24T10:16:51.17Z" }, + { url = "https://files.pythonhosted.org/packages/79/1d/f3803f12c8827cc1846eb79a2e2ddccf164d90833b9d56c829dc14b9f3e8/aqua_fetch-1.0.1-py3-none-any.whl", hash = "sha256:053ce30fcf9c76564a83dba1ded39e254f629a2d1cc08b3989093b7612a5a6aa", size = 289114, upload-time = "2026-02-11T08:02:54.233Z" }, ] [package.optional-dependencies] @@ -830,24 +830,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3e/8d/86586c0d75110f774e46e2bd6d134e2d1cca1dedc9bb08c388fa3df76acd/cftime-1.6.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a3cda6fd12c7fb25eff40a6a857a2bf4d03e8cc71f80485d8ddc65ccbd80f16a", size = 1718573, upload-time = "2025-10-13T18:56:02.788Z" }, { url = "https://files.pythonhosted.org/packages/bb/fe/7956914cfc135992e89098ebbc67d683c51ace5366ba4b114fef1de89b21/cftime-1.6.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:28cda78d685397ba23d06273b9c916c3938d8d9e6872a537e76b8408a321369b", size = 1788563, upload-time = "2025-10-13T18:56:04.075Z" }, { url = "https://files.pythonhosted.org/packages/e5/c7/6669708fcfe1bb7b2a7ce693b8cc67165eac00d3ac5a5e8f6ce1be551ff9/cftime-1.6.5-cp311-cp311-win_amd64.whl", hash = "sha256:93ead088e3a216bdeb9368733a0ef89a7451dfc1d2de310c1c0366a56ad60dc8", size = 473631, upload-time = "2025-10-13T18:56:05.159Z" }, + { url = "https://files.pythonhosted.org/packages/82/c5/d70cb1ab533ca790d7c9b69f98215fa4fead17f05547e928c8f2b8f96e54/cftime-1.6.5-cp311-cp311-win_arm64.whl", hash = "sha256:3384d69a0a7f3d45bded21a8cbcce66c8ba06c13498eac26c2de41b1b9b6e890", size = 459383, upload-time = "2026-01-02T21:16:47.317Z" }, { url = "https://files.pythonhosted.org/packages/b6/c1/e8cb7f78a3f87295450e7300ebaecf83076d96a99a76190593d4e1d2be40/cftime-1.6.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:eef25caed5ebd003a38719bd3ff8847cd52ef2ea56c3ebdb2c9345ba131fc7c5", size = 504175, upload-time = "2025-10-13T18:56:06.398Z" }, { url = "https://files.pythonhosted.org/packages/50/1a/86e1072b09b2f9049bb7378869f64b6747f96a4f3008142afed8955b52a4/cftime-1.6.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c87d2f3b949e45463e559233c69e6a9cf691b2b378c1f7556166adfabbd1c6b0", size = 485980, upload-time = "2025-10-13T18:56:08.669Z" }, { url = "https://files.pythonhosted.org/packages/35/28/d3177b60da3f308b60dee2aef2eb69997acfab1e863f0bf0d2a418396ce5/cftime-1.6.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:82cb413973cc51b55642b3a1ca5b28db5b93a294edbef7dc049c074b478b4647", size = 1591166, upload-time = "2025-10-13T19:39:14.109Z" }, { url = "https://files.pythonhosted.org/packages/d1/fd/a7266970312df65e68b5641b86e0540a739182f5e9c62eec6dbd29f18055/cftime-1.6.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:85ba8e7356d239cfe56ef7707ac30feaf67964642ac760a82e507ee3c5db4ac4", size = 1642614, upload-time = "2025-10-13T18:56:09.815Z" }, { url = "https://files.pythonhosted.org/packages/c4/73/f0035a4bc2df8885bb7bd5fe63659686ea1ec7d0cc74b4e3d50e447402e5/cftime-1.6.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:456039af7907a3146689bb80bfd8edabd074c7f3b4eca61f91b9c2670addd7ad", size = 1688090, upload-time = "2025-10-13T18:56:11.442Z" }, { url = "https://files.pythonhosted.org/packages/88/15/8856a0ab76708553ff597dd2e617b088c734ba87dc3fd395e2b2f3efffe8/cftime-1.6.5-cp312-cp312-win_amd64.whl", hash = "sha256:da84534c43699960dc980a9a765c33433c5de1a719a4916748c2d0e97a071e44", size = 464840, upload-time = "2025-10-13T18:56:12.506Z" }, + { url = "https://files.pythonhosted.org/packages/3a/85/451009a986d9273d2208fc0898aa00262275b5773259bf3f942f6716a9e7/cftime-1.6.5-cp312-cp312-win_arm64.whl", hash = "sha256:c62cd8db9ea40131eea7d4523691c5d806d3265d31279e4a58574a42c28acd77", size = 450534, upload-time = "2026-01-02T21:16:48.784Z" }, { url = "https://files.pythonhosted.org/packages/2e/60/74ea344b3b003fada346ed98a6899085d6fd4c777df608992d90c458fda6/cftime-1.6.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4aba66fd6497711a47c656f3a732c2d1755ad15f80e323c44a8716ebde39ddd5", size = 502453, upload-time = "2025-10-13T18:56:13.545Z" }, { url = "https://files.pythonhosted.org/packages/1e/14/adb293ac6127079b49ff11c05cf3d5ce5c1f17d097f326dc02d74ddfcb6e/cftime-1.6.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:89e7cba699242366e67d6fb5aee579440e791063f92a93853610c91647167c0d", size = 484541, upload-time = "2025-10-13T18:56:14.612Z" }, { url = "https://files.pythonhosted.org/packages/4f/74/bb8a4566af8d0ef3f045d56c462a9115da4f04b07c7fbbf2b4875223eebd/cftime-1.6.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2f1eb43d7a7b919ec99aee709fb62ef87ef1cf0679829ef93d37cc1c725781e9", size = 1591014, upload-time = "2025-10-13T19:39:15.346Z" }, { url = "https://files.pythonhosted.org/packages/ba/08/52f06ff2f04d376f9cd2c211aefcf2b37f1978e43289341f362fc99f6a0e/cftime-1.6.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e02a1d80ffc33fe469c7db68aa24c4a87f01da0c0c621373e5edadc92964900b", size = 1633625, upload-time = "2025-10-13T18:56:15.745Z" }, { url = "https://files.pythonhosted.org/packages/cf/33/03e0b23d58ea8fab94ecb4f7c5b721e844a0800c13694876149d98830a73/cftime-1.6.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:18ab754805233cdd889614b2b3b86a642f6d51a57a1ec327c48053f3414f87d8", size = 1684269, upload-time = "2025-10-13T18:56:17.04Z" }, { url = "https://files.pythonhosted.org/packages/a4/60/a0cfba63847b43599ef1cdbbf682e61894994c22b9a79fd9e1e8c7e9de41/cftime-1.6.5-cp313-cp313-win_amd64.whl", hash = "sha256:6c27add8f907f4a4cd400e89438f2ea33e2eb5072541a157a4d013b7dbe93f9c", size = 465364, upload-time = "2025-10-13T18:56:18.05Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e8/ec32f2aef22c15604e6fda39ff8d581a00b5469349f8fba61640d5358d2c/cftime-1.6.5-cp313-cp313-win_arm64.whl", hash = "sha256:31d1ff8f6bbd4ca209099d24459ec16dea4fb4c9ab740fbb66dd057ccbd9b1b9", size = 450468, upload-time = "2026-01-02T21:16:50.193Z" }, { url = "https://files.pythonhosted.org/packages/ea/6c/a9618f589688358e279720f5c0fe67ef0077fba07334ce26895403ebc260/cftime-1.6.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c69ce3bdae6a322cbb44e9ebc20770d47748002fb9d68846a1e934f1bd5daf0b", size = 502725, upload-time = "2025-10-13T18:56:19.424Z" }, { url = "https://files.pythonhosted.org/packages/d8/e3/da3c36398bfb730b96248d006cabaceed87e401ff56edafb2a978293e228/cftime-1.6.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e62e9f2943e014c5ef583245bf2e878398af131c97e64f8cd47c1d7baef5c4e2", size = 485445, upload-time = "2025-10-13T18:56:20.853Z" }, { url = "https://files.pythonhosted.org/packages/32/93/b05939e5abd14bd1ab69538bbe374b4ee2a15467b189ff895e9a8cdaddf6/cftime-1.6.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7da5fdaa4360d8cb89b71b8ded9314f2246aa34581e8105c94ad58d6102d9e4f", size = 1584434, upload-time = "2025-10-13T19:39:17.084Z" }, { url = "https://files.pythonhosted.org/packages/7f/89/648397f9936e0b330999c4e776ebf296ec3c6a65f9901687dbca4ab820da/cftime-1.6.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bff865b4ea4304f2744a1ad2b8149b8328b321dd7a2b9746ef926d229bd7cd49", size = 1609812, upload-time = "2025-10-13T18:56:21.971Z" }, { url = "https://files.pythonhosted.org/packages/e7/0f/901b4835aa67ad3e915605d4e01d0af80a44b114eefab74ae33de6d36933/cftime-1.6.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e552c5d1c8a58f25af7521e49237db7ca52ed2953e974fe9f7c4491e95fdd36c", size = 1669768, upload-time = "2025-10-13T18:56:24.027Z" }, { url = "https://files.pythonhosted.org/packages/22/d5/e605e4b28363e7a9ae98ed12cabbda5b155b6009270e6a231d8f10182a17/cftime-1.6.5-cp314-cp314-win_amd64.whl", hash = "sha256:e645b095dc50a38ac454b7e7f0742f639e7d7f6b108ad329358544a6ff8c9ba2", size = 463818, upload-time = "2025-10-13T18:56:25.376Z" }, + { url = "https://files.pythonhosted.org/packages/3d/89/a8f85ae697ff10206ec401c2621f5ca9f327554f586d62f244739ceeb347/cftime-1.6.5-cp314-cp314-win_arm64.whl", hash = "sha256:b9044d7ac82d3d8af189df1032fdc871bbd3f3dd41a6ec79edceb5029b71e6e0", size = 459862, upload-time = "2026-01-02T20:45:02.625Z" }, + { url = "https://files.pythonhosted.org/packages/ab/05/7410e12fd03a0c52717e74e6a1b49958810807dda212e23b65d43ea99676/cftime-1.6.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:9ef56460cb0576e1a9161e1428c9e1a633f809a23fa9d598f313748c1ae5064e", size = 533781, upload-time = "2026-01-02T20:45:04.818Z" }, + { url = "https://files.pythonhosted.org/packages/44/ba/10e3546426d3ed9f9cc82e4a99836bb6fac1642c7830f7bdd0ac1c3f0805/cftime-1.6.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:4f4873d38b10032f9f3111c547a1d485519ae64eee6a7a2d091f1f8b08e1ba50", size = 515218, upload-time = "2026-01-02T20:45:06.788Z" }, + { url = "https://files.pythonhosted.org/packages/bd/68/efa11eae867749e921bfec6a865afdba8166e96188112dde70bb8bb49254/cftime-1.6.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ccce0f4c9d3f38dd948a117e578b50d0e0db11e2ca9435fb358fd524813e4b61", size = 1579932, upload-time = "2026-01-02T20:45:11.194Z" }, + { url = "https://files.pythonhosted.org/packages/9d/6c/0971e602c1390a423e6621dfbad9f1d375186bdaf9c9c7f75e06f1fbf355/cftime-1.6.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:19cbfc5152fb0b34ce03acf9668229af388d7baa63a78f936239cb011ccbe6b1", size = 1555894, upload-time = "2026-01-02T20:45:16.351Z" }, + { url = "https://files.pythonhosted.org/packages/ad/fc/8475a15b7c3209a4a68b563dfc5e01ce74f2d8b9822372c3d30c68ab7f39/cftime-1.6.5-cp314-cp314t-win_amd64.whl", hash = "sha256:4470cd5ef3c2514566f53efbcbb64dd924fa0584637d90285b2f983bd4ee7d97", size = 513027, upload-time = "2026-01-02T20:45:20.023Z" }, + { url = "https://files.pythonhosted.org/packages/f7/80/4ecbda8318fbf40ad4e005a4a93aebba69e81382e5b4c6086251cd5d0ee8/cftime-1.6.5-cp314-cp314t-win_arm64.whl", hash = "sha256:034c15a67144a0a5590ef150c99f844897618b148b87131ed34fda7072614662", size = 469065, upload-time = "2026-01-02T20:45:23.398Z" }, ] [[package]] @@ -2231,6 +2241,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/91/ae2eb6b7979e2f9b035a9f612cf70f1bf54aad4e1d125129bef1eae96f19/greenlet-3.2.4-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c2ca18a03a8cfb5b25bc1cbe20f3d9a4c80d8c3b13ba3df49ac3961af0b1018d", size = 584358, upload-time = "2025-08-07T13:18:23.708Z" }, { url = "https://files.pythonhosted.org/packages/f7/85/433de0c9c0252b22b16d413c9407e6cb3b41df7389afc366ca204dbc1393/greenlet-3.2.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9fe0a28a7b952a21e2c062cd5756d34354117796c6d9215a87f55e38d15402c5", size = 1113550, upload-time = "2025-08-07T13:42:37.467Z" }, { url = "https://files.pythonhosted.org/packages/a1/8d/88f3ebd2bc96bf7747093696f4335a0a8a4c5acfcf1b757717c0d2474ba3/greenlet-3.2.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8854167e06950ca75b898b104b63cc646573aa5fef1353d4508ecdd1ee76254f", size = 1137126, upload-time = "2025-08-07T13:18:20.239Z" }, + { url = "https://files.pythonhosted.org/packages/f1/29/74242b7d72385e29bcc5563fba67dad94943d7cd03552bac320d597f29b2/greenlet-3.2.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f47617f698838ba98f4ff4189aef02e7343952df3a615f847bb575c3feb177a7", size = 1544904, upload-time = "2025-11-04T12:42:04.763Z" }, + { url = "https://files.pythonhosted.org/packages/c8/e2/1572b8eeab0f77df5f6729d6ab6b141e4a84ee8eb9bc8c1e7918f94eda6d/greenlet-3.2.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:af41be48a4f60429d5cad9d22175217805098a9ef7c40bfef44f7669fb9d74d8", size = 1611228, upload-time = "2025-11-04T12:42:08.423Z" }, { url = "https://files.pythonhosted.org/packages/d6/6f/b60b0291d9623c496638c582297ead61f43c4b72eef5e9c926ef4565ec13/greenlet-3.2.4-cp310-cp310-win_amd64.whl", hash = "sha256:73f49b5368b5359d04e18d15828eecc1806033db5233397748f4ca813ff1056c", size = 298654, upload-time = "2025-08-07T13:50:00.469Z" }, { url = "https://files.pythonhosted.org/packages/a4/de/f28ced0a67749cac23fecb02b694f6473f47686dff6afaa211d186e2ef9c/greenlet-3.2.4-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:96378df1de302bc38e99c3a9aa311967b7dc80ced1dcc6f171e99842987882a2", size = 272305, upload-time = "2025-08-07T13:15:41.288Z" }, { url = "https://files.pythonhosted.org/packages/09/16/2c3792cba130000bf2a31c5272999113f4764fd9d874fb257ff588ac779a/greenlet-3.2.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1ee8fae0519a337f2329cb78bd7a8e128ec0f881073d43f023c7b8d4831d5246", size = 632472, upload-time = "2025-08-07T13:42:55.044Z" }, @@ -2240,6 +2252,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1f/8e/abdd3f14d735b2929290a018ecf133c901be4874b858dd1c604b9319f064/greenlet-3.2.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2523e5246274f54fdadbce8494458a2ebdcdbc7b802318466ac5606d3cded1f8", size = 587684, upload-time = "2025-08-07T13:18:25.164Z" }, { url = "https://files.pythonhosted.org/packages/5d/65/deb2a69c3e5996439b0176f6651e0052542bb6c8f8ec2e3fba97c9768805/greenlet-3.2.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1987de92fec508535687fb807a5cea1560f6196285a4cde35c100b8cd632cc52", size = 1116647, upload-time = "2025-08-07T13:42:38.655Z" }, { url = "https://files.pythonhosted.org/packages/3f/cc/b07000438a29ac5cfb2194bfc128151d52f333cee74dd7dfe3fb733fc16c/greenlet-3.2.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:55e9c5affaa6775e2c6b67659f3a71684de4c549b3dd9afca3bc773533d284fa", size = 1142073, upload-time = "2025-08-07T13:18:21.737Z" }, + { url = "https://files.pythonhosted.org/packages/67/24/28a5b2fa42d12b3d7e5614145f0bd89714c34c08be6aabe39c14dd52db34/greenlet-3.2.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c9c6de1940a7d828635fbd254d69db79e54619f165ee7ce32fda763a9cb6a58c", size = 1548385, upload-time = "2025-11-04T12:42:11.067Z" }, + { url = "https://files.pythonhosted.org/packages/6a/05/03f2f0bdd0b0ff9a4f7b99333d57b53a7709c27723ec8123056b084e69cd/greenlet-3.2.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:03c5136e7be905045160b1b9fdca93dd6727b180feeafda6818e6496434ed8c5", size = 1613329, upload-time = "2025-11-04T12:42:12.928Z" }, { url = "https://files.pythonhosted.org/packages/d8/0f/30aef242fcab550b0b3520b8e3561156857c94288f0332a79928c31a52cf/greenlet-3.2.4-cp311-cp311-win_amd64.whl", hash = "sha256:9c40adce87eaa9ddb593ccb0fa6a07caf34015a29bf8d344811665b573138db9", size = 299100, upload-time = "2025-08-07T13:44:12.287Z" }, { url = "https://files.pythonhosted.org/packages/44/69/9b804adb5fd0671f367781560eb5eb586c4d495277c93bde4307b9e28068/greenlet-3.2.4-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3b67ca49f54cede0186854a008109d6ee71f66bd57bb36abd6d0a0267b540cdd", size = 274079, upload-time = "2025-08-07T13:15:45.033Z" }, { url = "https://files.pythonhosted.org/packages/46/e9/d2a80c99f19a153eff70bc451ab78615583b8dac0754cfb942223d2c1a0d/greenlet-3.2.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddf9164e7a5b08e9d22511526865780a576f19ddd00d62f8a665949327fde8bb", size = 640997, upload-time = "2025-08-07T13:42:56.234Z" }, @@ -2249,6 +2263,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/19/0d/6660d55f7373b2ff8152401a83e02084956da23ae58cddbfb0b330978fe9/greenlet-3.2.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b3812d8d0c9579967815af437d96623f45c0f2ae5f04e366de62a12d83a8fb0", size = 607586, upload-time = "2025-08-07T13:18:28.544Z" }, { url = "https://files.pythonhosted.org/packages/8e/1a/c953fdedd22d81ee4629afbb38d2f9d71e37d23caace44775a3a969147d4/greenlet-3.2.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:abbf57b5a870d30c4675928c37278493044d7c14378350b3aa5d484fa65575f0", size = 1123281, upload-time = "2025-08-07T13:42:39.858Z" }, { url = "https://files.pythonhosted.org/packages/3f/c7/12381b18e21aef2c6bd3a636da1088b888b97b7a0362fac2e4de92405f97/greenlet-3.2.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:20fb936b4652b6e307b8f347665e2c615540d4b42b3b4c8a321d8286da7e520f", size = 1151142, upload-time = "2025-08-07T13:18:22.981Z" }, + { url = "https://files.pythonhosted.org/packages/27/45/80935968b53cfd3f33cf99ea5f08227f2646e044568c9b1555b58ffd61c2/greenlet-3.2.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ee7a6ec486883397d70eec05059353b8e83eca9168b9f3f9a361971e77e0bcd0", size = 1564846, upload-time = "2025-11-04T12:42:15.191Z" }, + { url = "https://files.pythonhosted.org/packages/69/02/b7c30e5e04752cb4db6202a3858b149c0710e5453b71a3b2aec5d78a1aab/greenlet-3.2.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:326d234cbf337c9c3def0676412eb7040a35a768efc92504b947b3e9cfc7543d", size = 1633814, upload-time = "2025-11-04T12:42:17.175Z" }, { url = "https://files.pythonhosted.org/packages/e9/08/b0814846b79399e585f974bbeebf5580fbe59e258ea7be64d9dfb253c84f/greenlet-3.2.4-cp312-cp312-win_amd64.whl", hash = "sha256:a7d4e128405eea3814a12cc2605e0e6aedb4035bf32697f72deca74de4105e02", size = 299899, upload-time = "2025-08-07T13:38:53.448Z" }, { url = "https://files.pythonhosted.org/packages/49/e8/58c7f85958bda41dafea50497cbd59738c5c43dbbea5ee83d651234398f4/greenlet-3.2.4-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:1a921e542453fe531144e91e1feedf12e07351b1cf6c9e8a3325ea600a715a31", size = 272814, upload-time = "2025-08-07T13:15:50.011Z" }, { url = "https://files.pythonhosted.org/packages/62/dd/b9f59862e9e257a16e4e610480cfffd29e3fae018a68c2332090b53aac3d/greenlet-3.2.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd3c8e693bff0fff6ba55f140bf390fa92c994083f838fece0f63be121334945", size = 641073, upload-time = "2025-08-07T13:42:57.23Z" }, @@ -2258,6 +2274,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ee/43/3cecdc0349359e1a527cbf2e3e28e5f8f06d3343aaf82ca13437a9aa290f/greenlet-3.2.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23768528f2911bcd7e475210822ffb5254ed10d71f4028387e5a99b4c6699671", size = 610497, upload-time = "2025-08-07T13:18:31.636Z" }, { url = "https://files.pythonhosted.org/packages/b8/19/06b6cf5d604e2c382a6f31cafafd6f33d5dea706f4db7bdab184bad2b21d/greenlet-3.2.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:00fadb3fedccc447f517ee0d3fd8fe49eae949e1cd0f6a611818f4f6fb7dc83b", size = 1121662, upload-time = "2025-08-07T13:42:41.117Z" }, { url = "https://files.pythonhosted.org/packages/a2/15/0d5e4e1a66fab130d98168fe984c509249c833c1a3c16806b90f253ce7b9/greenlet-3.2.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:d25c5091190f2dc0eaa3f950252122edbbadbb682aa7b1ef2f8af0f8c0afefae", size = 1149210, upload-time = "2025-08-07T13:18:24.072Z" }, + { url = "https://files.pythonhosted.org/packages/1c/53/f9c440463b3057485b8594d7a638bed53ba531165ef0ca0e6c364b5cc807/greenlet-3.2.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6e343822feb58ac4d0a1211bd9399de2b3a04963ddeec21530fc426cc121f19b", size = 1564759, upload-time = "2025-11-04T12:42:19.395Z" }, + { url = "https://files.pythonhosted.org/packages/47/e4/3bb4240abdd0a8d23f4f88adec746a3099f0d86bfedb623f063b2e3b4df0/greenlet-3.2.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ca7f6f1f2649b89ce02f6f229d7c19f680a6238af656f61e0115b24857917929", size = 1634288, upload-time = "2025-11-04T12:42:21.174Z" }, { url = "https://files.pythonhosted.org/packages/0b/55/2321e43595e6801e105fcfdee02b34c0f996eb71e6ddffca6b10b7e1d771/greenlet-3.2.4-cp313-cp313-win_amd64.whl", hash = "sha256:554b03b6e73aaabec3745364d6239e9e012d64c68ccd0b8430c64ccc14939a8b", size = 299685, upload-time = "2025-08-07T13:24:38.824Z" }, { url = "https://files.pythonhosted.org/packages/22/5c/85273fd7cc388285632b0498dbbab97596e04b154933dfe0f3e68156c68c/greenlet-3.2.4-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:49a30d5fda2507ae77be16479bdb62a660fa51b1eb4928b524975b3bde77b3c0", size = 273586, upload-time = "2025-08-07T13:16:08.004Z" }, { url = "https://files.pythonhosted.org/packages/d1/75/10aeeaa3da9332c2e761e4c50d4c3556c21113ee3f0afa2cf5769946f7a3/greenlet-3.2.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:299fd615cd8fc86267b47597123e3f43ad79c9d8a22bebdce535e53550763e2f", size = 686346, upload-time = "2025-08-07T13:42:59.944Z" }, @@ -2265,6 +2283,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dc/8b/29aae55436521f1d6f8ff4e12fb676f3400de7fcf27fccd1d4d17fd8fecd/greenlet-3.2.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b4a1870c51720687af7fa3e7cda6d08d801dae660f75a76f3845b642b4da6ee1", size = 694659, upload-time = "2025-08-07T13:53:17.759Z" }, { url = "https://files.pythonhosted.org/packages/92/2e/ea25914b1ebfde93b6fc4ff46d6864564fba59024e928bdc7de475affc25/greenlet-3.2.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:061dc4cf2c34852b052a8620d40f36324554bc192be474b9e9770e8c042fd735", size = 695355, upload-time = "2025-08-07T13:18:34.517Z" }, { url = "https://files.pythonhosted.org/packages/72/60/fc56c62046ec17f6b0d3060564562c64c862948c9d4bc8aa807cf5bd74f4/greenlet-3.2.4-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44358b9bf66c8576a9f57a590d5f5d6e72fa4228b763d0e43fee6d3b06d3a337", size = 657512, upload-time = "2025-08-07T13:18:33.969Z" }, + { url = "https://files.pythonhosted.org/packages/23/6e/74407aed965a4ab6ddd93a7ded3180b730d281c77b765788419484cdfeef/greenlet-3.2.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2917bdf657f5859fbf3386b12d68ede4cf1f04c90c3a6bc1f013dd68a22e2269", size = 1612508, upload-time = "2025-11-04T12:42:23.427Z" }, + { url = "https://files.pythonhosted.org/packages/0d/da/343cd760ab2f92bac1845ca07ee3faea9fe52bee65f7bcb19f16ad7de08b/greenlet-3.2.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:015d48959d4add5d6c9f6c5210ee3803a830dce46356e3bc326d6776bde54681", size = 1680760, upload-time = "2025-11-04T12:42:25.341Z" }, { url = "https://files.pythonhosted.org/packages/e3/a5/6ddab2b4c112be95601c13428db1d8b6608a8b6039816f2ba09c346c08fc/greenlet-3.2.4-cp314-cp314-win_amd64.whl", hash = "sha256:e37ab26028f12dbb0ff65f29a8d3d44a765c61e729647bf2ddfbbed621726f01", size = 303425, upload-time = "2025-08-07T13:32:27.59Z" }, ] @@ -2346,8 +2366,7 @@ wheels = [ [[package]] name = "hydrodataset" -version = "0.2.0" -source = { registry = "https://pypi.org/simple" } +source = { editable = "../hydrodataset" } dependencies = [ { name = "aqua-fetch", extra = ["all"] }, { name = "bokeh" }, @@ -2371,10 +2390,55 @@ dependencies = [ { name = "zarr", version = "2.18.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "zarr", version = "3.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/25/ba/2a1f4dc463a0447b9e87f81222cdac1cdfd117ec991e293702ed39f58b02/hydrodataset-0.2.0.tar.gz", hash = "sha256:636c27ac76a2606d70cb974288005976833691c2c54576f88666ee38d8be149c", size = 66290, upload-time = "2025-10-30T07:40:39.767Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ea/d0/aa2fde0dc9b1824e872a77656c440cf50d735bd42cd408e8e2a31705c828/hydrodataset-0.2.0-py3-none-any.whl", hash = "sha256:2ea8ba0584c75e10f5238562aebd4e019a8af4fc7b3559cb4c6b9128c070d692", size = 103848, upload-time = "2025-10-30T07:40:38.328Z" }, + +[package.metadata] +requires-dist = [ + { name = "aqua-fetch", extras = ["all"], specifier = ">=1.0.1" }, + { name = "black", marker = "extra == 'dev'" }, + { name = "black", marker = "extra == 'lint'" }, + { name = "bokeh", specifier = ">=2.4.2" }, + { name = "build", marker = "extra == 'release'" }, + { name = "bump2version", marker = "extra == 'dev'" }, + { name = "codetiming" }, + { name = "coverage", marker = "extra == 'dev'" }, + { name = "dask", extras = ["complete"] }, + { name = "dask", extras = ["distributed"] }, + { name = "flake8", marker = "extra == 'dev'" }, + { name = "flake8", marker = "extra == 'lint'" }, + { name = "grip", marker = "extra == 'dev'" }, + { name = "grip", marker = "extra == 'docs'" }, + { name = "hatch-vcs", marker = "extra == 'release'" }, + { name = "hatchling", marker = "extra == 'release'" }, + { name = "hydroutils" }, + { name = "ipykernel", marker = "extra == 'docs'" }, + { name = "mkdocs-git-revision-date-plugin", marker = "extra == 'docs'" }, + { name = "mkdocs-jupyter", marker = "extra == 'docs'" }, + { name = "mkdocs-material", marker = "extra == 'docs'" }, + { name = "mkdocstrings", marker = "extra == 'docs'" }, + { name = "mkdocstrings-python", marker = "extra == 'docs'" }, + { name = "netcdf4" }, + { name = "openpyxl", specifier = ">=3.1.5" }, + { name = "pint" }, + { name = "pint-pandas" }, + { name = "pint-xarray" }, + { name = "py7zr", specifier = ">=1.0.0" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=3" }, + { name = "pytest-runner", marker = "extra == 'dev'" }, + { name = "requests", specifier = ">=2.32.4" }, + { name = "scipy" }, + { name = "sphinx", marker = "extra == 'dev'" }, + { name = "sphinx", marker = "extra == 'docs'" }, + { name = "tox", marker = "extra == 'dev'" }, + { name = "twine", marker = "extra == 'dev'" }, + { name = "twine", marker = "extra == 'release'" }, + { name = "tzfpy" }, + { name = "watchdog", marker = "extra == 'dev'" }, + { name = "wheel", marker = "extra == 'dev'" }, + { name = "xarray" }, + { name = "xlrd", specifier = ">=2.0.2" }, + { name = "zarr" }, ] +provides-extras = ["dev", "docs", "lint", "release"] [[package]] name = "hydrodatasource" @@ -2458,7 +2522,7 @@ requires-dist = [ { name = "flake8", marker = "extra == 'dev'" }, { name = "geopandas" }, { name = "h5py" }, - { name = "hydrodataset", specifier = ">=0.2.0" }, + { name = "hydrodataset", editable = "../hydrodataset" }, { name = "hydrotopo" }, { name = "hydroutils", specifier = ">=0.1.0" }, { name = "intake" }, From 931880a5be8f7a21df98aeae8b4e885d003e80d2 Mon Sep 17 00:00:00 2001 From: owen Date: Thu, 18 Jun 2026 09:32:45 +0800 Subject: [PATCH 14/34] feat: add configs/datasets.yml with hydrodatasource dataset registry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Register songliao_event → floodevent reader with relative path - Follows ADR 0001 format: dataset id → reader alias + relative path Co-Authored-By: Claude --- configs/datasets.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 configs/datasets.yml diff --git a/configs/datasets.yml b/configs/datasets.yml new file mode 100644 index 0000000..5d658e5 --- /dev/null +++ b/configs/datasets.yml @@ -0,0 +1,12 @@ +# Hydrodatasource dataset registry. +# Maps dataset id -> reader alias + relative path. +# Format: datasets..reader (str), datasets..path (relative str) +# +# Rules (from hydromodel ADR 0001): +# - path must be relative (no absolute, no URI, no .. traversal) +# - reader must name a registered alias in data_resolver.READER_ALIASES + +datasets: + songliao_event: + reader: floodevent + path: projects/songliao/event From 0fce21fa0e90aa9d8de6b53fdd84cb4cc8d1cf39 Mon Sep 17 00:00:00 2001 From: owen Date: Thu, 18 Jun 2026 09:38:38 +0800 Subject: [PATCH 15/34] =?UTF-8?q?refactor:=20clean=20up=20config.py=20?= =?UTF-8?q?=E2=80=94=20remove=20DB/minio,=20add=20new=20APIs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove postgres globals (POSTGRES_PARAM, PS, get_postgres_connection) - Remove minio clients unused externally (RO, S3, MC) - Remove unused constants (STATION_BUCKET, STATION_OBJECT, GRID_INTERIM_BUCKET) - Remove minio/postgres validation from read_setting() - Add bridge: translate new storage.* format → legacy local_data_path keys - Add pure-function APIs: get_local_root(), get_cache_dir(), get_storage_config() - Delegate to hydrodataset.configs.settings for new-format resolution Co-Authored-By: Claude --- hydrodatasource/configs/config.py | 382 ++++++++++++++---------------- 1 file changed, 179 insertions(+), 203 deletions(-) diff --git a/hydrodatasource/configs/config.py b/hydrodatasource/configs/config.py index e520321..b78608f 100644 --- a/hydrodatasource/configs/config.py +++ b/hydrodatasource/configs/config.py @@ -1,232 +1,208 @@ """ -Author: Jianfeng Zhu +Author: Jianfeng Zhu, Wenyu Ouyang Date: 2023-10-25 18:49:02 -LastEditTime: 2025-10-31 11:28:17 +LastEditTime: 2025-06-18 LastEditors: Wenyu Ouyang -Description: Some configs for minio server -FilePath: \hydrodatasource\hydrodatasource\configs\config.py -Copyright (c) 2023-2024 Wenyu Ouyang. All rights reserved. +Description: Storage and cache configuration for hydrodatasource. + +Reads ~/hydro_setting.yml (shared with hydromodel and hydrodataset). +Supports the unified storage.* format defined in hydromodel ADR 0001. + +Legacy minio.* and postgres.* config blocks have been removed. +Cloud access is unified under storage.s3. +Database access is decoupled to a separate real-time service. + +Critical globals (SETTING, CACHE_DIR, FS, LOCAL_DATA_PATH, MINIO_PARAM) are +initialized at import time for backward compatibility. +New code should use the pure-function APIs (get_local_root, get_cache_dir). + +FilePath: \\hydrodatasource\\hydrodatasource\\configs\\config.py +Copyright (c) 2023-2026 Wenyu Ouyang. All rights reserved. """ import os from pathlib import Path -import boto3 +from typing import Any, Dict, Optional + import s3fs import yaml -from minio import Minio -import psycopg2 +from hydrodataset.configs.settings import ( + get_cache_dir as _hd_get_cache_dir, + get_local_root as _hd_get_local_root, + get_storage_config as _hd_get_storage_config, +) + +# ── Internal helpers (defined first — called by init) ─────────────────────── -def read_setting(setting_path): + +def _load_settings_from_file() -> Dict[str, Any]: + """Load settings from ~/hydro_setting.yml (supports old and new formats).""" + setting_path = os.path.join(Path.home(), "hydro_setting.yml") if not os.path.exists(setting_path): - raise FileNotFoundError(f"Configuration file not found: {setting_path}") + return {} - with open(setting_path, "r", encoding="utf-8") as file: # 指定编码为 UTF-8 + with open(setting_path, "r", encoding="utf-8") as file: setting = yaml.safe_load(file) - example_setting = ( - "minio:\n" - " server_url: 'http://minio.waterism.com:9090' # Update with your URL\n" - " client_endpoint: 'http://minio.waterism.com:9000' # Update with your URL\n" - " access_key: 'your minio access key'\n" - " secret: 'your minio secret'\n\n" - "local_data_path:\n" - " root: 'D:\\data\\waterism' # Update with your root data directory\n" - " datasets-origin: 'D:\\data\\waterism\\datasets-origin'\n" - " datasets-interim: 'D:\\data\\waterism\\datasets-interim'\n" - " cache: 'D:\\data\\waterism\\.cache'\n" - "postgres:\n" - " server_url: your_postgres_server_url\n" - " port: 5432\n" - " username: your_postgres_username\n" - " password: your_postgres_secret_code\n" - " database: your_postgres_database\n" - ) - if setting is None: - raise ValueError( - f"Configuration file is empty or has invalid format.\n\nExample configuration:\n{example_setting}" - ) + return {} - # Define the expected structure - # Note: cache is now optional - expected_structure = { - "minio": ["server_url", "client_endpoint", "access_key", "secret"], - "local_data_path": ["root", "datasets-origin", "datasets-interim"], - "postgres": ["server_url", "port", "username", "password", "database"], - } + return setting - # Validate the structure - try: - for key, subkeys in expected_structure.items(): - if key not in setting: - raise KeyError(f"Missing required key in config: {key}") - - if isinstance(subkeys, list): - for subkey in subkeys: - if subkey not in setting[key]: - raise KeyError(f"Missing required subkey '{subkey}' in '{key}'") - except KeyError as e: - raise ValueError( - f"Incorrect configuration format: {e}\n\nExample configuration:\n{example_setting}" - ) from e - return setting +# ── Module-level state (initialized once at import for backward compat) ── + +SETTING: Dict[str, Any] = {} +CACHE_DIR: str = "" +LOCAL_DATA_PATH: str = "" +FS: Optional[s3fs.S3FileSystem] = None +MINIO_PARAM: Dict[str, str] = {} + +def _init_settings() -> None: + """Load settings from ~/hydro_setting.yml once at module load time.""" + global SETTING, CACHE_DIR, LOCAL_DATA_PATH, FS, MINIO_PARAM -SETTING_FILE = os.path.join(Path.home(), "hydro_setting.yml") -try: - SETTING = read_setting(SETTING_FILE) - LOCAL_DATA_PATH = SETTING["local_data_path"]["root"] - # cache is optional, use default if not provided - CACHE_DIR = SETTING["local_data_path"].get("cache", Path.home().joinpath("hydrodatasource_data", ".cache")) -except (ValueError, FileNotFoundError) as e: - LOCAL_DATA_PATH = Path.home().joinpath("hydrodatasource_data") - CACHE_DIR = Path.home().joinpath("hydrodatasource_data", ".cache") - SETTING = { - "minio": { - "server_url": "", - "client_endpoint": "", - "access_key": "", - "secret": "", - }, - "postgres": { - "server_url": "", - "port": 5432, - "username": "", - "password": "", - "database": "", - }, - "local_data_path": { - "root": LOCAL_DATA_PATH, - "cache": CACHE_DIR, + try: + setting = _load_settings_from_file() + except (ValueError, FileNotFoundError) as e: + print(f"Warning: Could not load hydro_setting.yml: {e}") + setting = {} + + default_root = os.path.join(Path.home(), "hydrodatasource_data") + if not setting: + print(f"Using default data paths in home directory: {default_root}") + setting = { + "local_data_path": { + "root": default_root, + "datasets-origin": os.path.join(default_root, "datasets-origin"), + "datasets-interim": os.path.join(default_root, "datasets-interim"), + "cache": os.path.join(default_root, ".cache"), + } } - } - print(e) -except Exception as e: - LOCAL_DATA_PATH = Path.home().joinpath("hydrodatasource_data") - CACHE_DIR = Path.home().joinpath("hydrodatasource_data", ".cache") - SETTING = { - "minio": { - "server_url": "", - "client_endpoint": "", - "access_key": "", - "secret": "", - }, - "postgres": { - "server_url": "", - "port": 5432, - "username": "", - "password": "", - "database": "", - }, - "local_data_path": { - "root": LOCAL_DATA_PATH, - "cache": CACHE_DIR, + + # Bridge: if new format (storage.*) exists but old (local_data_path) is + # missing, create derived local_data_path for backward compat. + if "local_data_path" not in setting and "storage" in setting: + storage_local = setting["storage"].get("local", {}) + storage_root = storage_local.get("root", default_root) + setting["local_data_path"] = { + "root": storage_root, + "datasets-origin": os.path.join(storage_root, "datasets-origin"), + "datasets-interim": os.path.join(storage_root, "datasets-interim"), + "cache": storage_local.get( + "cache", os.path.join(storage_root, ".cache") + ), } - } - print(f"Unexpected error: {e}") - -# Initialize remote service settings -MINIO_PARAM = {} -RO = {} -S3 = None -MC = None -PS = None -FS = None - -STATION_BUCKET = "stations" -STATION_OBJECT = "sites.csv" -GRID_INTERIM_BUCKET = "grids-interim" - -# Handle MinIO connection -try: - # MinIO parameters - MINIO_PARAM = { - "endpoint_url": SETTING["minio"]["client_endpoint"], - "key": SETTING["minio"]["access_key"], - "secret": SETTING["minio"]["secret"], - } - - # Initialize S3 FileSystem - FS = s3fs.S3FileSystem( - client_kwargs={"endpoint_url": MINIO_PARAM["endpoint_url"]}, - key=MINIO_PARAM["key"], - secret=MINIO_PARAM["secret"], - use_ssl=False, - ) - # remote_options parameters for xr open_dataset from minio - RO = { - "client_kwargs": {"endpoint_url": MINIO_PARAM["endpoint_url"]}, - "key": MINIO_PARAM["key"], - "secret": MINIO_PARAM["secret"], - "use_ssl": False, - } - - # Set up MinIO client - S3 = boto3.client( - "s3", - endpoint_url=SETTING["minio"]["client_endpoint"], - aws_access_key_id=MINIO_PARAM["key"], - aws_secret_access_key=MINIO_PARAM["secret"], - ) - MC = Minio( - SETTING["minio"]["client_endpoint"].replace("http://", ""), - access_key=MINIO_PARAM["key"], - secret_key=MINIO_PARAM["secret"], - secure=False, # True if using HTTPS + root = setting.get("local_data_path", {}).get("root", default_root) + cache = setting.get("local_data_path", {}).get( + "cache", os.path.join(Path.home(), "hydrodatasource_data", ".cache") ) -except KeyError as e: - print(f"Configuration error: {e}") -except Exception as e: - print(f"Remote service setup failed: {e}") - -# PostgreSQL connection parameters (lazy loading) -POSTGRES_PARAM = {} -PS = None - -# Initialize PostgreSQL parameters but don't connect yet -try: - POSTGRES_PARAM = { - "database": SETTING["postgres"]["database"], - "user": SETTING["postgres"]["username"], - "password": SETTING["postgres"]["password"], - "host": SETTING["postgres"]["server_url"], - "port": SETTING["postgres"]["port"], - } -except KeyError as e: - print(f"PostgreSQL configuration error: {e}") - POSTGRES_PARAM = {} - - -def get_postgres_connection(): - """ - Get PostgreSQL connection with lazy initialization. - - Returns: - psycopg2.connection: PostgreSQL connection object, or None if failed - """ - global PS - if PS is not None: - # Check if connection is still alive - try: - PS.cursor().execute("SELECT 1") - return PS - except (psycopg2.OperationalError, psycopg2.InterfaceError): - # Connection is dead, need to reconnect - PS = None - - # Create new connection - if POSTGRES_PARAM: + SETTING = setting + LOCAL_DATA_PATH = root + CACHE_DIR = cache + + # Initialize S3FS only if minio credentials are present in old-format config + minio_cfg = setting.get("minio", {}) + if minio_cfg.get("client_endpoint") and minio_cfg.get("access_key"): + MINIO_PARAM = { + "endpoint_url": minio_cfg["client_endpoint"], + "key": minio_cfg["access_key"], + "secret": minio_cfg.get("secret", ""), + } try: - PS = psycopg2.connect(**POSTGRES_PARAM) - return PS + FS = s3fs.S3FileSystem( + client_kwargs={"endpoint_url": MINIO_PARAM["endpoint_url"]}, + key=MINIO_PARAM["key"], + secret=MINIO_PARAM["secret"], + use_ssl=False, + ) except Exception as e: - print(f"Failed to connect to PostgreSQL: {e}") - PS = None - return None + print(f"Warning: S3FS initialization failed: {e}") + FS = None else: - print("PostgreSQL configuration not available") - return None + MINIO_PARAM = {} + FS = None + + +# Initialize at module load (backward compat) +_init_settings() + + +# ── Public API (pure functions — no side effects) ───────────────────────── + + +def get_local_root() -> Optional[Path]: + """Get the local storage root directory. + + Uses the new storage.local.root format (via hydrodataset settings). + Falls back to the old local_data_path.root format for backward compat. + """ + root = _hd_get_local_root() + if root is not None: + return root + if LOCAL_DATA_PATH: + return Path(LOCAL_DATA_PATH) + return None + + +def get_cache_dir() -> Path: + """Get cache directory via hydrodataset's cache resolution logic.""" + return _hd_get_cache_dir() + + +def get_storage_config() -> Dict[str, Any]: + """Get storage configuration block (new format).""" + return _hd_get_storage_config() + + +def read_setting(setting_path: str) -> Dict[str, Any]: + """Read and validate a hydro_setting.yml file. + + Accepts both old (local_data_path.*) and new (storage.*) formats. + No longer requires minio or postgres sections. + """ + if not os.path.exists(setting_path): + raise FileNotFoundError(f"Configuration file not found: {setting_path}") + + with open(setting_path, "r", encoding="utf-8") as file: + setting = yaml.safe_load(file) + + example_setting = ( + "# New format (recommended):\n" + "storage:\n" + " default_source: local\n" + " local:\n" + " root: 'D:\\\\data\\\\hydrodatasource'\n" + " s3:\n" + " bucket: hydro-data\n" + " prefix: hydromodel\n" + " region: us-east-1\n" + " profile: default\n\n" + "# Old format (deprecated, still supported):\n" + "local_data_path:\n" + " root: 'D:\\\\data\\\\waterism'\n" + " datasets-origin: 'D:\\\\data\\\\waterism\\\\datasets-origin'\n" + " datasets-interim: 'D:\\\\data\\\\waterism\\\\datasets-interim'\n" + " cache: 'D:\\\\data\\\\waterism\\\\.cache'\n" + ) + + if setting is None: + raise ValueError( + f"Configuration file is empty or has invalid format.\n\n" + f"Example configuration:\n{example_setting}" + ) + + has_old = "local_data_path" in setting + has_new = "storage" in setting + if not has_old and not has_new: + raise ValueError( + f"Configuration must have 'storage' (new format) or " + f"'local_data_path' (old format) section.\n\n" + f"Example configuration:\n{example_setting}" + ) + + return setting From 30385c82a02d563fce2bb1a41190a58f9401d4a9 Mon Sep 17 00:00:00 2001 From: owen Date: Thu, 18 Jun 2026 09:47:59 +0800 Subject: [PATCH 16/34] feat: add URI-accepting constructors to all 11 reader classes - HydroData base: accept optional uri kwarg, bypass os.path.join() - All subclasses pass uri through to super().__init__() - Fix FloodEventDatasource: remove leftover download=False kwarg - Add pathlib.Path import to data_source.py - All classes verified via inspect.signature check Co-Authored-By: Claude --- hydrodatasource/reader/data_source.py | 104 ++++++++++++++----- hydrodatasource/reader/floodevent.py | 6 +- hydrodatasource/reader/gages.py | 4 +- hydrodatasource/reader/grdc.py | 4 +- hydrodatasource/reader/rainfall_reader.py | 4 +- hydrodatasource/reader/reservoir_datasets.py | 4 +- hydrodatasource/reader/rsvr_inflow_reader.py | 4 +- 7 files changed, 93 insertions(+), 37 deletions(-) diff --git a/hydrodatasource/reader/data_source.py b/hydrodatasource/reader/data_source.py index 4bd002f..584c466 100644 --- a/hydrodatasource/reader/data_source.py +++ b/hydrodatasource/reader/data_source.py @@ -2,6 +2,7 @@ import glob import json import os +from pathlib import Path import re import numpy as np import pandas as pd @@ -34,15 +35,28 @@ class HydroData(ABC): """An interface for reading multi-modal data sources. + Supports two construction patterns: + 1. New (resolved URI): HydroData(uri="/path/to/data") + 2. Legacy: HydroData(data_path="/parent", dataset_name="child") + Parameters ---------- - ABC : _type_ - _description_ + data_path : str, optional + Parent directory path (legacy mode). + dataset_name : str, optional + Subdirectory name under data_path (legacy mode). + uri : str, optional + Resolved absolute URI pointing directly to the data directory. + When provided, data_path and dataset_name are ignored. """ - def __init__(self, data_path, dataset_name): - self.data_source_dir = os.path.join(data_path, dataset_name) - self.dataset_name = dataset_name + def __init__(self, data_path=None, dataset_name=None, *, uri=None): + if uri is not None: + self.data_source_dir = str(uri) + self.dataset_name = dataset_name or Path(str(uri)).name + else: + self.data_source_dir = os.path.join(data_path, dataset_name) + self.dataset_name = dataset_name def get_name(self): raise NotImplementedError @@ -64,28 +78,45 @@ class SelfMadeHydroDataset(HydroData): Only two directories are needed: attributes and timeseries """ - def __init__(self, data_path, dataset_name, time_unit=None, **kwargs): + def __init__( + self, + data_path=None, + dataset_name=None, + time_unit=None, + *, + uri=None, + **kwargs, + ): """Initialize a self-made Caravan-style dataset. Parameters ---------- - data_path : str - The path to the custom-made data sources' parent directory. - dataset_name : str - SelfMadeHydroDataset's name, for example, googleflood or fdsources, - different dataset may use this same datasource class, but they have different dataset_name. + data_path : str, optional + Parent directory path (legacy mode). + dataset_name : str, optional + Dataset name (legacy mode, or label when uri is used). time_unit : list, optional - we have different time units, by default None + Time units to process, by default None. + uri : str, optional + Resolved absolute URI pointing directly to the data directory. + When provided, data_path and dataset_name are used as labels only. kwargs : dict, optional - additional keyword arguments, by default None + Additional keyword arguments, by default None. """ - if any(unit not in ["1h", "3h", "1D", "8D", "1M"] for unit in time_unit): + if time_unit and any( + unit not in ["1h", "3h", "1D", "8D", "1M"] for unit in time_unit + ): raise ValueError( "time_unit must be one of ['1h', '3h', '1D', '8D','1M']. We only support these time units now." ) - # TODO: maybe starting with "s3://" is a better idea? - self.head = "minio" if "s3://" in data_path else "local" - super().__init__(data_path, dataset_name) + # Determine filesystem head from URI or data_path + effective_path = str(uri) if uri is not None else (data_path or "") + self.head = "minio" if "s3://" in effective_path else "local" + super().__init__( + data_path=data_path, + dataset_name=dataset_name, + uri=uri, + ) self.data_source_description = self.set_data_source_describe() self.camels_sites = self.read_site_info() self.time_unit = time_unit @@ -852,12 +883,18 @@ def read_mean_prcp(self, gage_id_lst=None, unit="mm/d"): class LongTermDataset(SelfMadeHydroDataset): - def __init__(self, data_path, download=False, time_unit=None, **kwargs): + def __init__(self, data_path=None, download=False, time_unit=None, *, uri=None, **kwargs): if time_unit is None: time_unit = ["1M"] if "dataset_name" not in kwargs: kwargs["dataset_name"] = "gmspa" - super().__init__(data_path, download, time_unit, **kwargs) + super().__init__( + data_path=data_path, + dataset_name=kwargs.pop("dataset_name", str(download)), + time_unit=time_unit, + uri=uri, + **kwargs, + ) def set_data_source_describe(self): the_dict = super().set_data_source_describe() @@ -912,7 +949,7 @@ def cache_global_dataset(self, object_ids: list = None, t_range_list: list = Non class SelfMadeForecastDataset(SelfMadeHydroDataset): """For selfmadehydrodataset, we design a new file format for forecast data from GFS et al.""" - def __init__(self, data_path, dataset_name, time_unit=None): + def __init__(self, data_path=None, dataset_name=None, time_unit=None, *, uri=None): """intialize a Class for reading forecast data Parameters @@ -924,7 +961,12 @@ def __init__(self, data_path, dataset_name, time_unit=None): dataset_name: str name will be used for cache files """ - super().__init__(data_path, dataset_name, time_unit) + super().__init__( + data_path=data_path, + dataset_name=dataset_name, + time_unit=time_unit, + uri=uri, + ) def set_data_source_describe(self): """set data source description @@ -1321,7 +1363,7 @@ class StationHydroDataset(SelfMadeHydroDataset): - adjacency_xxx_True.csv """ - def __init__(self, data_path, dataset_name, time_unit=None, **kwargs): + def __init__(self, data_path=None, dataset_name=None, time_unit=None, *, uri=None, **kwargs): """Initialize StationHydroDataset. Parameters @@ -1335,7 +1377,13 @@ def __init__(self, data_path, dataset_name, time_unit=None, **kwargs): **kwargs : dict Additional keyword arguments passed to parent class """ - super().__init__(data_path, dataset_name, time_unit, **kwargs) + super().__init__( + data_path=data_path, + dataset_name=dataset_name, + time_unit=time_unit, + uri=uri, + **kwargs, + ) self.station_info = None self.basin_station_mapping = None @@ -1919,7 +1967,7 @@ class TgHydroDatasource(SelfMadeHydroDataset): └── shapes/ # 区间流域形状文件 """ - def __init__(self, data_path, dataset_name=None, time_unit=None, **kwargs): + def __init__(self, data_path=None, dataset_name=None, time_unit=None, *, uri=None, **kwargs): """ 初始化TG流域数据集 @@ -1941,7 +1989,13 @@ def __init__(self, data_path, dataset_name=None, time_unit=None, **kwargs): # "inter_basin_pred_file is required; please run scripts/generate_inter_basin_predictions.py to generate the inter-basin predictions." # ) # kwargs.pop("inter_basin_pred_file") - super().__init__(data_path, dataset_name, time_unit, **kwargs) + super().__init__( + data_path=data_path, + dataset_name=dataset_name, + time_unit=time_unit, + uri=uri, + **kwargs, + ) # 基于 intermediate 属性数据的 downstream_id 生成拓扑关系 self.dg = self._build_graph_from_intermediate_attributes() diff --git a/hydrodatasource/reader/floodevent.py b/hydrodatasource/reader/floodevent.py index 8136ddf..86f2ad6 100644 --- a/hydrodatasource/reader/floodevent.py +++ b/hydrodatasource/reader/floodevent.py @@ -32,9 +32,11 @@ class FloodEventDatasource(SelfMadeHydroDataset): def __init__( self, - data_path: str, + data_path: str = None, dataset_name: str = "songliaorrevents", time_unit: Optional[List[str]] = None, + *, + uri: str = None, rain_key: str = "rain", pet_key: str = "ES", net_rain_key: str = "net_rain", @@ -86,9 +88,9 @@ def __init__( super().__init__( data_path=data_path, - download=False, time_unit=time_unit, dataset_name=dataset_name, + uri=uri, **kwargs, ) diff --git a/hydrodatasource/reader/gages.py b/hydrodatasource/reader/gages.py index bc43dbb..69badc8 100644 --- a/hydrodatasource/reader/gages.py +++ b/hydrodatasource/reader/gages.py @@ -25,8 +25,8 @@ class Gages(HydroData): - def __init__(self, data_path, dataset_name): - super().__init__(data_path, dataset_name) + def __init__(self, data_path=None, dataset_name=None, *, uri=None): + super().__init__(data_path=data_path, dataset_name=dataset_name, uri=uri) self.data_source_description = self.set_data_source_describe() self.gages_sites = self.read_site_info() diff --git a/hydrodatasource/reader/grdc.py b/hydrodatasource/reader/grdc.py index 287f197..da49308 100644 --- a/hydrodatasource/reader/grdc.py +++ b/hydrodatasource/reader/grdc.py @@ -28,8 +28,8 @@ class Grdc(HydroData): """Reading GRDC streamflow data.""" - def __init__(self, data_path): - self.data_source_dir = data_path + def __init__(self, data_path=None, *, uri=None): + self.data_source_dir = str(uri) if uri is not None else data_path self.data_source_description = self.set_data_source_describe() self.grdc_site_info = self.read_site_info() diff --git a/hydrodatasource/reader/rainfall_reader.py b/hydrodatasource/reader/rainfall_reader.py index c2f33ee..df7edb0 100644 --- a/hydrodatasource/reader/rainfall_reader.py +++ b/hydrodatasource/reader/rainfall_reader.py @@ -81,7 +81,7 @@ def filter_out_rows(df1, df2, key_columns): @hydro_logger class RainfallReader(HydroData): - def __init__(self, data_folder): + def __init__(self, data_folder=None, *, uri=None): """A reader for basin rainfall data Even for station data, we still use basin as the index @@ -90,7 +90,7 @@ def __init__(self, data_folder): data_folder : _type_ _description_ """ - self.data_source_dir = data_folder + self.data_source_dir = str(uri) if uri is not None else data_folder self.data_source_description = self.set_data_source_describe() self.station_info = self.read_station_info() diff --git a/hydrodatasource/reader/reservoir_datasets.py b/hydrodatasource/reader/reservoir_datasets.py index cedf2bd..6fad3dc 100644 --- a/hydrodatasource/reader/reservoir_datasets.py +++ b/hydrodatasource/reader/reservoir_datasets.py @@ -17,8 +17,8 @@ class Crd(HydroData): """Reading CRD reservoir dataset""" - def __init__(self, data_path): - self.data_source_dir = data_path + def __init__(self, data_path=None, *, uri=None): + self.data_source_dir = str(uri) if uri is not None else data_path self.data_source_description = self.set_data_source_describe() self.reservoir_info = self.read_reservoir_info() diff --git a/hydrodatasource/reader/rsvr_inflow_reader.py b/hydrodatasource/reader/rsvr_inflow_reader.py index 3eeaae0..311eb23 100644 --- a/hydrodatasource/reader/rsvr_inflow_reader.py +++ b/hydrodatasource/reader/rsvr_inflow_reader.py @@ -18,8 +18,8 @@ class RsvrInflowReader(HydroData): - def __init__(self, data_folder): - self.data_source_dir = data_folder + def __init__(self, data_folder=None, *, uri=None): + self.data_source_dir = str(uri) if uri is not None else data_folder self.data_source_description = self.set_data_source_describe() self.rsvr_info = self.read_rsvr_info() From 3550269ea15e80854f2fe15ef0969d5da73ba931 Mon Sep 17 00:00:00 2001 From: owen Date: Thu, 18 Jun 2026 09:49:04 +0800 Subject: [PATCH 17/34] feat: export READER_ALIASES, resolve_data_path, DatasetResolutionError from package Co-Authored-By: Claude --- hydrodatasource/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/hydrodatasource/__init__.py b/hydrodatasource/__init__.py index 7795fda..012e30a 100644 --- a/hydrodatasource/__init__.py +++ b/hydrodatasource/__init__.py @@ -7,3 +7,10 @@ # Fallback version if _version.py doesn't exist yet __version__ = "0.0.0+unknown" __version_tuple__ = (0, 0, 0, "unknown") + +# Re-export config APIs for convenience +from .configs.data_resolver import ( + READER_ALIASES, + resolve_data_path, + DatasetResolutionError, +) From e9f806580ac4585d2d0e78121153b9db3859db0c Mon Sep 17 00:00:00 2001 From: owen Date: Tue, 23 Jun 2026 09:54:01 +0800 Subject: [PATCH 18/34] test: add 51 unit tests for URI constructors, config API, and package exports - test_reader_uri_constructors.py: 27 tests covering HydroData base, Pattern A (7 subclasses via inspect.signature), Pattern B (4 subclasses via mock), and SelfMadeHydroDataset real instantiation with tmp_path - test_config_api.py: 17 tests for read_setting validation, get_local_root/get_cache_dir/get_storage_config APIs, backward-compat globals, and format bridging - test_package_exports.py: 7 tests verifying READER_ALIASES, resolve_data_path, and DatasetResolutionError are importable from hydrodatasource top-level package, with merged alias counts All 70 new tests (51 new + 19 existing test_data_resolver.py) pass GREEN. --- tests/test_config_api.py | 287 +++++++++++++++++++ tests/test_package_exports.py | 67 +++++ tests/test_reader_uri_constructors.py | 398 ++++++++++++++++++++++++++ 3 files changed, 752 insertions(+) create mode 100644 tests/test_config_api.py create mode 100644 tests/test_package_exports.py create mode 100644 tests/test_reader_uri_constructors.py diff --git a/tests/test_config_api.py b/tests/test_config_api.py new file mode 100644 index 0000000..f1ffbc3 --- /dev/null +++ b/tests/test_config_api.py @@ -0,0 +1,287 @@ +"""Tests for hydrodatasource.configs.config — new public API functions. + +Tests the pure-function APIs (get_local_root, get_cache_dir, get_storage_config) +and read_setting() validation, plus backward-compatible globals. + +These tests verify the changes made in feat/unified-data-interface. +""" + +import os +import tempfile +from pathlib import Path +from unittest.mock import patch + +import pytest +import yaml + + +# ── read_setting() tests ──────────────────────────────────────────────────── + + +class TestReadSetting: + """Verify read_setting() accepts both old and new config formats.""" + + def test_new_format_storage_section(self, tmp_path): + """New format with storage.* sections is accepted.""" + from hydrodatasource.configs.config import read_setting + + setting_path = tmp_path / "hydro_setting.yml" + setting_path.write_text( + yaml.dump( + { + "storage": { + "default_source": "local", + "local": {"root": str(tmp_path / "data")}, + "s3": { + "bucket": "my-bucket", + "prefix": "hydromodel", + }, + } + } + ), + encoding="utf-8", + ) + + result = read_setting(str(setting_path)) + assert "storage" in result + assert result["storage"]["s3"]["bucket"] == "my-bucket" + + def test_old_format_local_data_path(self, tmp_path): + """Old format with local_data_path.* is accepted (backward compat).""" + from hydrodatasource.configs.config import read_setting + + setting_path = tmp_path / "hydro_setting.yml" + setting_path.write_text( + yaml.dump( + { + "local_data_path": { + "root": str(tmp_path / "data"), + "datasets-origin": str(tmp_path / "data" / "datasets-origin"), + "datasets-interim": str(tmp_path / "data" / "datasets-interim"), + } + } + ), + encoding="utf-8", + ) + + result = read_setting(str(setting_path)) + assert "local_data_path" in result + + def test_both_formats_accepted(self, tmp_path): + """Config with both old and new sections is valid.""" + from hydrodatasource.configs.config import read_setting + + setting_path = tmp_path / "hydro_setting.yml" + setting_path.write_text( + yaml.dump( + { + "storage": { + "local": {"root": str(tmp_path / "data")}, + }, + "local_data_path": { + "root": str(tmp_path / "data"), + }, + } + ), + encoding="utf-8", + ) + + result = read_setting(str(setting_path)) + assert "storage" in result + assert "local_data_path" in result + + def test_missing_file_raises(self, tmp_path): + """Missing setting file raises FileNotFoundError.""" + from hydrodatasource.configs.config import read_setting + + with pytest.raises(FileNotFoundError, match="not found"): + read_setting(str(tmp_path / "nonexistent.yml")) + + def test_empty_setting_raises(self, tmp_path): + """Empty/null setting raises ValueError.""" + from hydrodatasource.configs.config import read_setting + + setting_path = tmp_path / "hydro_setting.yml" + setting_path.write_text("", encoding="utf-8") + + with pytest.raises(ValueError, match="empty"): + read_setting(str(setting_path)) + + def test_missing_required_sections_raises(self, tmp_path): + """Setting without 'storage' or 'local_data_path' raises ValueError.""" + from hydrodatasource.configs.config import read_setting + + setting_path = tmp_path / "hydro_setting.yml" + setting_path.write_text( + yaml.dump({"some_other_key": "value"}), encoding="utf-8" + ) + + with pytest.raises(ValueError, match="must have 'storage'"): + read_setting(str(setting_path)) + + +# ── Pure-function API tests ───────────────────────────────────────────────── + + +class TestGetLocalRoot: + """Verify get_local_root() resolves correctly.""" + + def test_returns_path_when_local_data_path_set(self, monkeypatch): + """get_local_root returns LOCAL_DATA_PATH when not None.""" + from hydrodatasource.configs import config + + monkeypatch.setattr(config, "LOCAL_DATA_PATH", "/test/root") + # Patch _hd_get_local_root to return None (no new format) + monkeypatch.setattr(config, "_hd_get_local_root", lambda: None) + + result = config.get_local_root() + assert result == Path("/test/root") + + def test_returns_hd_root_when_available(self, monkeypatch): + """get_local_root prefers hydrodataset's root when available.""" + from hydrodatasource.configs import config + + monkeypatch.setattr( + config, "_hd_get_local_root", lambda: Path("/hd/root") + ) + monkeypatch.setattr(config, "LOCAL_DATA_PATH", "/old/root") + + result = config.get_local_root() + assert result == Path("/hd/root") + + def test_returns_none_when_nothing_configured(self, monkeypatch): + """get_local_root returns None when nothing is configured.""" + from hydrodatasource.configs import config + + monkeypatch.setattr(config, "_hd_get_local_root", lambda: None) + monkeypatch.setattr(config, "LOCAL_DATA_PATH", "") + + result = config.get_local_root() + assert result is None + + +class TestGetCacheDir: + """Verify get_cache_dir() delegates correctly.""" + + def test_returns_path(self, monkeypatch): + """get_cache_dir delegates to hydrodataset.""" + from hydrodatasource.configs import config + + monkeypatch.setattr( + config, "_hd_get_cache_dir", lambda: Path("/cache/dir") + ) + + result = config.get_cache_dir() + assert result == Path("/cache/dir") + + +class TestGetStorageConfig: + """Verify get_storage_config() returns storage config dict.""" + + def test_returns_dict(self, monkeypatch): + """get_storage_config delegates to hydrodataset.""" + from hydrodatasource.configs import config + + expected = { + "storage": { + "s3": {"bucket": "test-bucket", "prefix": "test-prefix"}, + } + } + monkeypatch.setattr(config, "_hd_get_storage_config", lambda: expected) + + result = config.get_storage_config() + assert result == expected + + +# ── Backward-compatible globals ────────────────────────────────────────────── + + +class TestBackwardCompatGlobals: + """Verify legacy globals are still accessible after import.""" + + def test_setting_is_dict(self): + """SETTING is a dict (may be empty if no config file).""" + from hydrodatasource.configs.config import SETTING + + assert isinstance(SETTING, dict) + + def test_local_data_path_is_str(self): + """LOCAL_DATA_PATH is a str (may be empty).""" + from hydrodatasource.configs.config import LOCAL_DATA_PATH + + assert isinstance(LOCAL_DATA_PATH, str) + + def test_cache_dir_is_str(self): + """CACHE_DIR is a str (may be empty).""" + from hydrodatasource.configs.config import CACHE_DIR + + assert isinstance(CACHE_DIR, str) + + def test_minio_param_is_dict(self): + """MINIO_PARAM is a dict (may be empty).""" + from hydrodatasource.configs.config import MINIO_PARAM + + assert isinstance(MINIO_PARAM, dict) + + +# ── Format bridging tests ─────────────────────────────────────────────────── + + +class TestFormatBridging: + """Verify old-format and new-format interop in _init_settings.""" + + def test_new_format_derives_local_data_path(self, monkeypatch, tmp_path): + """When only storage.* is set, local_data_path is derived.""" + from hydrodatasource.configs import config + import importlib + + # Create a temp setting file with only new format + setting_path = Path.home() / "hydro_setting.yml" + # We can't modify the real home config, so test the logic directly + # by calling _init_settings with mocked _load_settings_from_file + data_root = tmp_path / "data" + + original_load = config._load_settings_from_file + + def mock_load(): + return { + "storage": { + "default_source": "local", + "local": {"root": str(data_root)}, + } + } + + monkeypatch.setattr(config, "_load_settings_from_file", mock_load) + config._init_settings() + + assert config.LOCAL_DATA_PATH == str(data_root) + + # Restore + monkeypatch.setattr(config, "_load_settings_from_file", original_load) + config._init_settings() + + def test_old_format_still_works(self, monkeypatch, tmp_path): + """Old local_data_path format still initializes correctly.""" + from hydrodatasource.configs import config + + data_root = str(tmp_path / "data") + + def mock_load(): + return { + "local_data_path": { + "root": data_root, + "datasets-origin": os.path.join(data_root, "datasets-origin"), + "datasets-interim": os.path.join(data_root, "datasets-interim"), + "cache": os.path.join(data_root, ".cache"), + } + } + + original_load = config._load_settings_from_file + monkeypatch.setattr(config, "_load_settings_from_file", mock_load) + config._init_settings() + + assert config.LOCAL_DATA_PATH == data_root + + # Restore + monkeypatch.setattr(config, "_load_settings_from_file", original_load) + config._init_settings() diff --git a/tests/test_package_exports.py b/tests/test_package_exports.py new file mode 100644 index 0000000..0d29654 --- /dev/null +++ b/tests/test_package_exports.py @@ -0,0 +1,67 @@ +"""Tests for hydrodatasource package-level exports (__init__.py). + +Verifies that READER_ALIASES, resolve_data_path, and DatasetResolutionError +are importable from the top-level package. +""" + +import pytest + + +class TestPackageExports: + """Verify new exports in hydrodatasource.__init__.""" + + def test_reader_aliases_importable_from_package(self): + """READER_ALIASES is importable from hydrodatasource.""" + from hydrodatasource import READER_ALIASES + + assert isinstance(READER_ALIASES, dict) + assert len(READER_ALIASES) >= 11 + + def test_resolve_data_path_importable_from_package(self): + """resolve_data_path is importable from hydrodatasource.""" + from hydrodatasource import resolve_data_path + + assert callable(resolve_data_path) + + def test_dataset_resolution_error_importable_from_package(self): + """DatasetResolutionError is importable from hydrodatasource.""" + from hydrodatasource import DatasetResolutionError + + assert issubclass(DatasetResolutionError, ValueError) + + def test_merged_aliases_include_hydrodataset_entries(self): + """Merged READER_ALIASES includes entries from hydrodataset.""" + from hydrodatasource import READER_ALIASES + + # hydrodataset entries should be present (these are from the 37 public datasets) + assert "camels_us" in READER_ALIASES, "hydrodataset aliases not merged" + hd_entry = READER_ALIASES["camels_us"] + assert hd_entry["category"] == "hydrodataset" + + def test_merged_aliases_include_hydrodatasource_entries(self): + """Merged READER_ALIASES includes entries from hydrodatasource.""" + from hydrodatasource import READER_ALIASES + + assert "floodevent" in READER_ALIASES + hds_entry = READER_ALIASES["floodevent"] + assert hds_entry["category"] == "hydrodatasource" + + def test_hydrodatasource_aliases_total_count(self): + """hydrodatasource has exactly 11 reader aliases.""" + from hydrodatasource import READER_ALIASES + + hds_aliases = { + k: v for k, v in READER_ALIASES.items() if v.get("category") == "hydrodatasource" + } + assert len(hds_aliases) == 11, ( + f"Expected 11 hydrodatasource aliases, got {len(hds_aliases)}: " + f"{sorted(hds_aliases.keys())}" + ) + + def test_merged_total_count(self): + """Total merged aliases: 11 hydrodatasource + 33 hydrodataset = 44.""" + from hydrodatasource import READER_ALIASES + + assert len(READER_ALIASES) == 44, ( + f"Expected 44 total aliases (11 HDS + 33 HD), got {len(READER_ALIASES)}" + ) diff --git a/tests/test_reader_uri_constructors.py b/tests/test_reader_uri_constructors.py new file mode 100644 index 0000000..4d8711d --- /dev/null +++ b/tests/test_reader_uri_constructors.py @@ -0,0 +1,398 @@ +"""Tests for URI-accepting constructors in all 11 reader classes. + +Covers both construction patterns: + - New: ReaderClass(uri="/path/to/data") + - Legacy: ReaderClass(data_path="/parent", dataset_name="child") + +These tests verify the changes made in feat/unified-data-interface. +""" + +import os +import tempfile +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + + +# ── HydroData base class (no file I/O in __init__, can test directly) ────── + + +class TestHydroDataBase: + """Test the HydroData abstract base class constructor.""" + + def test_uri_construction_sets_data_source_dir(self): + """uri kwarg sets data_source_dir directly.""" + from hydrodatasource.reader.data_source import HydroData + + ds = HydroData(uri="/tmp/test/dataset") + assert ds.data_source_dir == "/tmp/test/dataset" + + def test_uri_construction_sets_dataset_name_from_uri(self): + """dataset_name defaults to the last component of the URI path.""" + from hydrodatasource.reader.data_source import HydroData + + ds = HydroData(uri="/tmp/test/my_dataset") + assert ds.dataset_name == "my_dataset" + + def test_uri_with_explicit_dataset_name(self): + """Explicit dataset_name is preserved when uri is provided.""" + from hydrodatasource.reader.data_source import HydroData + + ds = HydroData(uri="/tmp/test/data", dataset_name="custom_label") + assert ds.data_source_dir == "/tmp/test/data" + assert ds.dataset_name == "custom_label" + + def test_uri_as_pathlib_path(self): + """uri accepts pathlib.Path objects (via str conversion).""" + from hydrodatasource.reader.data_source import HydroData + + ds = HydroData(uri=Path("/tmp/test/dataset")) + assert ds.data_source_dir.endswith("dataset") + + def test_uri_with_s3_prefix(self): + """uri works with s3:// URIs.""" + from hydrodatasource.reader.data_source import HydroData + + ds = HydroData(uri="s3://bucket/prefix/dataset") + assert ds.data_source_dir == "s3://bucket/prefix/dataset" + assert ds.dataset_name == "dataset" + + def test_legacy_data_path_and_dataset_name(self): + """Legacy construction still works with data_path + dataset_name.""" + from hydrodatasource.reader.data_source import HydroData + + ds = HydroData(data_path="/parent", dataset_name="child") + assert ds.data_source_dir == os.path.join("/parent", "child") + assert ds.dataset_name == "child" + + def test_uri_takes_precedence_over_data_path(self): + """When uri is provided, data_path and dataset_name are ignored.""" + from hydrodatasource.reader.data_source import HydroData + + ds = HydroData( + uri="/explicit/uri/path", + data_path="/ignored/parent", + dataset_name="ignored_name", + ) + assert ds.data_source_dir == "/explicit/uri/path" + assert ds.dataset_name == "ignored_name" + + +# ── Pattern B classes (bypass super().__init__, set data_source_dir directly) ─ + + +class TestPatternBConstructors: + """Test classes that bypass super().__init__(). + + Grdc, RainfallReader, Crd, RsvrInflowReader — these set + data_source_dir directly and DON'T call super().__init__(). + """ + + @pytest.fixture + def mock_read_site_info(self): + """Mock all post-init methods that touch the filesystem. + + Pattern B classes call set_data_source_describe() and read_*_info() + after setting data_source_dir. Both need mocking. + """ + with patch( + "hydrodatasource.reader.grdc.Grdc.set_data_source_describe", + return_value={}, + ), patch( + "hydrodatasource.reader.grdc.Grdc.read_site_info", + return_value=MagicMock(), + ), patch( + "hydrodatasource.reader.rainfall_reader.RainfallReader.set_data_source_describe", + return_value={}, + ), patch( + "hydrodatasource.reader.rainfall_reader.RainfallReader.read_station_info", + return_value=MagicMock(), + ), patch( + "hydrodatasource.reader.reservoir_datasets.Crd.set_data_source_describe", + return_value={}, + ), patch( + "hydrodatasource.reader.reservoir_datasets.Crd.read_reservoir_info", + return_value=MagicMock(), + ), patch( + "hydrodatasource.reader.rsvr_inflow_reader.RsvrInflowReader.set_data_source_describe", + return_value={}, + ), patch( + "hydrodatasource.reader.rsvr_inflow_reader.RsvrInflowReader.read_rsvr_info", + return_value=MagicMock(), + ): + yield + + def test_grdc_uri_construction(self, mock_read_site_info): + """Grdc accepts uri kwarg and sets data_source_dir.""" + from hydrodatasource.reader.grdc import Grdc + + ds = Grdc(uri="/data/grdc") + assert ds.data_source_dir == "/data/grdc" + + def test_grdc_legacy_construction(self, mock_read_site_info): + """Grdc still works with legacy data_path arg.""" + from hydrodatasource.reader.grdc import Grdc + + ds = Grdc(data_path="/data/grdc") + assert ds.data_source_dir == "/data/grdc" + + def test_rainfall_reader_uri_construction(self, mock_read_site_info): + """RainfallReader accepts uri kwarg.""" + from hydrodatasource.reader.rainfall_reader import RainfallReader + + ds = RainfallReader(uri="/data/rainfall") + assert ds.data_source_dir == "/data/rainfall" + + def test_rainfall_reader_legacy_construction(self, mock_read_site_info): + """RainfallReader still works with legacy data_folder arg.""" + from hydrodatasource.reader.rainfall_reader import RainfallReader + + ds = RainfallReader(data_folder="/data/rainfall") + assert ds.data_source_dir == "/data/rainfall" + + def test_crd_uri_construction(self, mock_read_site_info): + """Crd accepts uri kwarg.""" + from hydrodatasource.reader.reservoir_datasets import Crd + + ds = Crd(uri="/data/crd") + assert ds.data_source_dir == "/data/crd" + + def test_crd_legacy_construction(self, mock_read_site_info): + """Crd still works with legacy data_path arg.""" + from hydrodatasource.reader.reservoir_datasets import Crd + + ds = Crd(data_path="/data/crd") + assert ds.data_source_dir == "/data/crd" + + def test_rsvr_inflow_uri_construction(self, mock_read_site_info): + """RsvrInflowReader accepts uri kwarg.""" + from hydrodatasource.reader.rsvr_inflow_reader import RsvrInflowReader + + ds = RsvrInflowReader(uri="/data/rsvr_inflow") + assert ds.data_source_dir == "/data/rsvr_inflow" + + def test_rsvr_inflow_legacy_construction(self, mock_read_site_info): + """RsvrInflowReader still works with legacy data_folder arg.""" + from hydrodatasource.reader.rsvr_inflow_reader import RsvrInflowReader + + ds = RsvrInflowReader(data_folder="/data/rsvr_inflow") + assert ds.data_source_dir == "/data/rsvr_inflow" + + +# ── Pattern A classes (call super().__init__ via SelfMadeHydroDataset) ────── + + +class TestPatternAConstructors: + """Test classes that inherit from SelfMadeHydroDataset. + + These classes call super().__init__() → HydroData.__init__() and + then initialize additional state (set_data_source_describe, read_site_info). + """ + + @pytest.fixture + def mock_hydrodata_init(self): + """Mock HydroData.__init__ and file-reading methods. + + We mock HydroData.__init__ so we don't need actual data directories. + This isolates the test to just the URI parameter plumbing. + """ + with patch( + "hydrodatasource.reader.data_source.HydroData.__init__", + return_value=None, + ), patch.object( + SelfMadeHydroDataset, + "set_data_source_describe", + return_value={}, + ), patch.object( + SelfMadeHydroDataset, + "read_site_info", + return_value=MagicMock(), + ), patch.object( + SelfMadeHydroDataset, + "get_name", + return_value="test", + ): + yield + + def test_selfmade_uri_accepted(self): + """SelfMadeHydroDataset accepts uri kwarg without crashing.""" + from hydrodatasource.reader.data_source import SelfMadeHydroDataset + + # Without mocking, we need a valid config. Let's just verify + # the signature accepts 'uri' by checking it doesn't raise TypeError. + import inspect + + sig = inspect.signature(SelfMadeHydroDataset.__init__) + params = list(sig.parameters.keys()) + assert "uri" in params + + def test_floodevent_uri_accepted(self): + """FloodEventDatasource accepts uri kwarg.""" + from hydrodatasource.reader.floodevent import FloodEventDatasource + import inspect + + sig = inspect.signature(FloodEventDatasource.__init__) + params = list(sig.parameters.keys()) + assert "uri" in params + + def test_longterm_uri_accepted(self): + """LongTermDataset accepts uri kwarg.""" + from hydrodatasource.reader.data_source import LongTermDataset + import inspect + + sig = inspect.signature(LongTermDataset.__init__) + params = list(sig.parameters.keys()) + assert "uri" in params + + def test_forecast_uri_accepted(self): + """SelfMadeForecastDataset accepts uri kwarg.""" + from hydrodatasource.reader.data_source import SelfMadeForecastDataset + import inspect + + sig = inspect.signature(SelfMadeForecastDataset.__init__) + params = list(sig.parameters.keys()) + assert "uri" in params + + def test_station_uri_accepted(self): + """StationHydroDataset accepts uri kwarg.""" + from hydrodatasource.reader.data_source import StationHydroDataset + import inspect + + sig = inspect.signature(StationHydroDataset.__init__) + params = list(sig.parameters.keys()) + assert "uri" in params + + def test_tghydro_uri_accepted(self): + """TgHydroDatasource accepts uri kwarg.""" + from hydrodatasource.reader.data_source import TgHydroDatasource + import inspect + + sig = inspect.signature(TgHydroDatasource.__init__) + params = list(sig.parameters.keys()) + assert "uri" in params + + def test_gages_uri_accepted(self): + """Gages accepts uri kwarg.""" + from hydrodatasource.reader.gages import Gages + import inspect + + sig = inspect.signature(Gages.__init__) + params = list(sig.parameters.keys()) + assert "uri" in params + + +# ── SelfMadeHydroDataset constructor edge cases ───────────────────────────── + + +class TestSelfMadeHydroDatasetConstructor: + """Test SelfMadeHydroDataset constructor edge cases. + + Uses a temporary directory with minimal structure to test real instantiation. + """ + + @pytest.fixture + def minimal_dataset(self, tmp_path): + """Create a minimal dataset directory with required structure.""" + data_dir = tmp_path / "minimal_dataset" + attrs_dir = data_dir / "attributes" + ts_dir = data_dir / "timeseries" / "1D" + attrs_dir.mkdir(parents=True) + ts_dir.mkdir(parents=True) + + # Create minimal attributes.csv + import pandas as pd + + attr_df = pd.DataFrame( + { + "basin_id": ["basin_001"], + "area": [100.0], + "p_mean": [800.0], + } + ) + attr_df.to_csv(attrs_dir / "attributes.csv", index=False) + + # Create a minimal timeseries file + import xarray as xr + import numpy as np + + ds = xr.Dataset( + { + "streamflow": xr.DataArray( + np.random.rand(10), + dims=["time"], + coords={ + "time": xr.date_range( + "2020-01-01", periods=10, freq="D" + ) + }, + ) + } + ) + ds.to_netcdf(ts_dir / "basin_001.nc") + + return data_dir + + def test_uri_construction_with_minimal_data(self, minimal_dataset): + """SelfMadeHydroDataset with uri can instantiate on real (minimal) data.""" + from hydrodatasource.reader.data_source import SelfMadeHydroDataset + + ds = SelfMadeHydroDataset( + uri=str(minimal_dataset), + time_unit=["1D"], + ) + assert ds.data_source_dir == str(minimal_dataset) + # dataset_name defaults to the directory name + assert ds.dataset_name == "minimal_dataset" + + def test_uri_construction_sets_head_local(self, minimal_dataset): + """head is 'local' when uri is a local path.""" + from hydrodatasource.reader.data_source import SelfMadeHydroDataset + + ds = SelfMadeHydroDataset( + uri=str(minimal_dataset), + time_unit=["1D"], + ) + assert ds.head == "local" + + def test_uri_construction_sets_head_minio(self): + """head is 'minio' when uri is an s3:// path.""" + from hydrodatasource.reader.data_source import SelfMadeHydroDataset + + # This will fail on read_site_info (no actual s3 data), but we can + # verify the head is set correctly before that happens. + try: + ds = SelfMadeHydroDataset( + uri="s3://bucket/prefix/dataset", + time_unit=["1D"], + ) + assert ds.head == "minio" + except (FileNotFoundError, OSError, ValueError): + # Expected — no actual data at the s3 URI in test + pass + + def test_time_unit_validation(self): + """Invalid time_unit raises ValueError.""" + from hydrodatasource.reader.data_source import SelfMadeHydroDataset + + with pytest.raises(ValueError, match="time_unit must be one of"): + SelfMadeHydroDataset( + uri="/some/path", + time_unit=["invalid_unit"], + ) + + def test_legacy_construction_with_minimal_data(self, minimal_dataset): + """Legacy data_path + dataset_name still works.""" + from hydrodatasource.reader.data_source import SelfMadeHydroDataset + + ds = SelfMadeHydroDataset( + data_path=str(minimal_dataset.parent), + dataset_name="minimal_dataset", + time_unit=["1D"], + ) + assert ds.dataset_name == "minimal_dataset" + assert str(minimal_dataset) in ds.data_source_dir + + +# ── Import fix: needed for patch.object ───────────────────────────────────── +from hydrodatasource.reader.data_source import SelfMadeHydroDataset From 16ceb2c39f849f29d6a38bfcf86f26e0f2e56d83 Mon Sep 17 00:00:00 2001 From: owen Date: Tue, 23 Jun 2026 15:22:50 +0800 Subject: [PATCH 19/34] fix: align data_resolver with hydrodataset public API, fix S3/os.path/return bugs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - data_resolver.py: drop private _load_registry import, use in-memory _HDS_DATASETS + extra_registry_dicts injection. Add local_root passthrough. Remove _load_yaml/_validate_relative_path/parent.parent.parent path chain. Net reduction: 164 → 118 lines (-28%). - floodevent.py: skip os.path.exists with self.head != 'minio' guard for S3/cloud URIs. Fix return {} → return None (2 sites) to avoid empty dict passing is-not-None gate. - grdc.py/reservoir_datasets.py/rainfall_reader.py/rsvr_inflow_reader.py: add self.dataset_name in Pattern B readers that bypass super().__init__(). - tests: add test_resolve_with_local_root_override, TestS3UriPathCheck, test_pattern_b_readers_have_dataset_name (4 parametrized). Fix mock paths for get_local_root/get_storage_config to hydrodataset namespace. --- hydrodatasource/configs/data_resolver.py | 202 +++---------------- hydrodatasource/reader/floodevent.py | 57 +++--- hydrodatasource/reader/grdc.py | 1 + hydrodatasource/reader/rainfall_reader.py | 2 + hydrodatasource/reader/reservoir_datasets.py | 2 + hydrodatasource/reader/rsvr_inflow_reader.py | 2 + tests/test_data_resolver.py | 21 +- tests/test_reader_uri_constructors.py | 67 ++++++ 8 files changed, 152 insertions(+), 202 deletions(-) diff --git a/hydrodatasource/configs/data_resolver.py b/hydrodatasource/configs/data_resolver.py index 7c02267..b5b4845 100644 --- a/hydrodatasource/configs/data_resolver.py +++ b/hydrodatasource/configs/data_resolver.py @@ -4,8 +4,10 @@ Defines READER_ALIASES for all hydrodatasource reader classes and provides resolve_data_path() following the same contract as hydromodel's ADR 0001. -Reuses hydrodataset's storage configuration parsing (settings.py) and -validation logic, so all three repos produce the same resolved URIs. +This module is a thin wrapper around hydrodataset.configs.data_resolver. +It only uses the PUBLIC API of hydrodataset (resolve_data_path, READER_ALIASES, +DatasetResolutionError). The hydrodatasource-specific parts are injected via +extra_registry_dicts and extra_reader_aliases parameters. Usage: from hydrodatasource.configs.data_resolver import ( @@ -14,22 +16,20 @@ DatasetResolutionError, ) - uri = resolve_data_path("songliao_event", project_root=".") - ds = FloodEventDatasource(data_path=uri, dataset_name="myevents") + uri = resolve_data_path("songliao_event", local_root="/data/hydro") + ds = FloodEventDatasource(uri=uri) """ from __future__ import annotations -from pathlib import Path, PurePosixPath, PureWindowsPath -from typing import Any, Dict, Optional - -import yaml +from pathlib import Path +from typing import Dict, Optional from hydrodataset.configs.data_resolver import ( DatasetResolutionError, READER_ALIASES as _HD_READER_ALIASES, + resolve_data_path as _hd_resolve_data_path, ) -from hydrodataset.configs.settings import get_local_root, get_storage_config # ── hydrodatasource-specific reader aliases ────────────────────────────── @@ -97,101 +97,14 @@ **_HDS_READER_ALIASES, } -FORBIDDEN_PATH_PATTERNS = {"://", ".."} - - -def _load_yaml(path: Path) -> Dict[str, Any]: - """Load a YAML file, returning {} if it does not exist.""" - if not path.exists(): - return {} - with path.open("r", encoding="utf-8") as f: - loaded = yaml.safe_load(f) - return loaded or {} - - -def _load_registry(project_root: Path) -> Dict[str, Dict[str, Any]]: - """Load dataset registry, checking hydrodatasource first then hydrodataset. - - Looks in these locations: - 1. {project_root}/configs/datasets.yml (project-level, takes precedence) - 2. hydrodatasource package's own configs/datasets.yml - 3. hydrodataset/configs/datasets.yml (package fallback) - - Parameters - ---------- - project_root : Path - Root of the calling project. - - Returns - ------- - dict - Dataset registry mapping dataset_id -> {'reader': ..., 'path': ...} - - Raises - ------ - DatasetResolutionError - If no registry file is found. - """ - # Project-level config (takes precedence) - project_registry = project_root / "configs" / "datasets.yml" - # hydrodatasource package-internal fallback - package_registry = ( - Path(__file__).resolve().parent.parent.parent / "configs" / "datasets.yml" - ) - # hydrodataset's own datasets.yml - hd_registry = ( - Path(__file__).resolve().parent.parent.parent.parent - / "hydrodataset" - / "configs" - / "datasets.yml" - ) - - registry_path = None - if project_registry.exists(): - registry_path = project_registry - elif package_registry.exists(): - registry_path = package_registry - elif hd_registry.exists(): - registry_path = hd_registry +# ── In-memory dataset registry (no YAML file needed) ───────────────────── - if registry_path is None: - raise DatasetResolutionError( - "Dataset registry not found. Tried:\n" - f" - {project_registry}\n" - f" - {package_registry}\n" - f" - {hd_registry}\n" - "Create configs/datasets.yml in your project." - ) - - data = _load_yaml(registry_path) - datasets = data.get("datasets") - if not isinstance(datasets, dict): - raise DatasetResolutionError( - f"Dataset registry in {registry_path} must have a 'datasets' mapping." - ) - return datasets - - -def _validate_relative_path(path_value: str, dataset_id: str) -> None: - """Ensure path is a safe relative path (no URI schemes, no .., no absolute).""" - if not isinstance(path_value, str): - raise DatasetResolutionError( - f"Dataset '{dataset_id}' path must be a string, " - f"got {type(path_value)}" - ) - for forbidden in FORBIDDEN_PATH_PATTERNS: - if forbidden in path_value: - raise DatasetResolutionError( - f"Dataset '{dataset_id}' path must be relative, " - f"not contain '{forbidden}'" - ) - windows_path = PureWindowsPath(path_value) - posix_path = PurePosixPath(path_value) - if windows_path.is_absolute() or posix_path.is_absolute(): - raise DatasetResolutionError( - f"Dataset '{dataset_id}' path must be relative, " - f"got absolute: '{path_value}'" - ) +_HDS_DATASETS: Dict[str, Dict[str, str]] = { + "songliao_event": { + "reader": "floodevent", + "path": "projects/songliao/event", + }, +} def resolve_data_path( @@ -199,12 +112,12 @@ def resolve_data_path( *, source: str = "local", project_root: Optional[str] = None, + local_root: Optional[str] = None, ) -> str: """Resolve a dataset id to an absolute data path (URI). - Combines the dataset registry entry with storage configuration - to produce a single absolute path. Follows the same contract as - hydromodel's resolve_data_cfgs and hydrodataset's resolve_data_path. + Thin wrapper around hydrodataset's resolve_data_path that injects + hydrodatasource-specific datasets and reader aliases. Parameters ---------- @@ -215,6 +128,9 @@ def resolve_data_path( project_root : str, optional Root of the calling project (for finding configs/datasets.yml). Defaults to current working directory. + local_root : str, optional + Override storage.local.root for project-level root override. + When provided, skips reading storage.local.root from settings. Returns ------- @@ -226,68 +142,12 @@ def resolve_data_path( DatasetResolutionError If any resolution step fails. """ - if source not in {"local", "cloud"}: - raise DatasetResolutionError( - f"source must be 'local' or 'cloud', got '{source}'" - ) - - root = Path(project_root) if project_root else Path.cwd() - registry = _load_registry(root) - - if dataset_id not in registry: - known = ", ".join(sorted(registry)) - raise DatasetResolutionError( - f"Unknown dataset id '{dataset_id}'. " - f"Known datasets: {known}" - ) - - dataset_spec = registry[dataset_id] - reader = dataset_spec.get("reader") - if not reader: - raise DatasetResolutionError( - f"Dataset '{dataset_id}' must define 'reader'" - ) - if reader not in READER_ALIASES: - raise DatasetResolutionError( - f"Unknown reader alias '{reader}' for dataset '{dataset_id}'" - ) - - relative_path = dataset_spec.get("path") - if not relative_path: - raise DatasetResolutionError( - f"Dataset '{dataset_id}' must define 'path'" - ) - _validate_relative_path(relative_path, dataset_id) - - if source == "local": - root_dir = get_local_root() - if root_dir is None: - raise DatasetResolutionError( - "storage.local.root is not configured. " - "Set it in ~/hydro_setting.yml" - ) - if not root_dir.exists(): - raise DatasetResolutionError( - f"storage.local.root does not exist: {root_dir}" - ) - resolved = root_dir / relative_path - if not resolved.exists(): - raise DatasetResolutionError( - f"Resolved dataset path does not exist: {resolved}" - ) - return str(resolved) - - # cloud source - storage = get_storage_config() - s3 = storage.get("s3") - if not isinstance(s3, dict): - raise DatasetResolutionError( - "storage.s3 is required for cloud source" - ) - bucket = s3.get("bucket") - if not bucket: - raise DatasetResolutionError("storage.s3.bucket is required") - prefix = str(s3.get("prefix") or "").strip("/") - rel = relative_path.replace("\\", "/").strip("/") - path = f"{prefix}/{rel}" if prefix else rel - return f"s3://{bucket}/{path}" + result = _hd_resolve_data_path( + dataset_id, + source=source, + project_root=project_root, + local_root=Path(local_root) if local_root else None, + extra_registry_dicts=[_HDS_DATASETS], + extra_reader_aliases=_HDS_READER_ALIASES, + ) + return str(result) diff --git a/hydrodatasource/reader/floodevent.py b/hydrodatasource/reader/floodevent.py index 86f2ad6..c3ef5e4 100644 --- a/hydrodatasource/reader/floodevent.py +++ b/hydrodatasource/reader/floodevent.py @@ -94,33 +94,34 @@ def __init__( **kwargs, ) - # 验证数据目录是否存在 - if not os.path.exists(self.data_source_dir): - raise FileNotFoundError( - f"❌ 数据集目录不存在: {self.data_source_dir}\n" - f"\n配置信息:" - f"\n data_path = {data_path}" - f"\n dataset_name = {dataset_name}" - f"\n 拼接后路径 = {self.data_source_dir}" - f"\n\n请检查:" - f"\n 1. data_path 是否指向数据集的父目录" - f"\n 2. dataset_name 拼写是否正确" - f"\n 3. 数据集目录是否实际存在" - ) + # 验证数据目录是否存在（S3/云端路径跳过本地文件系统检查） + if self.head != "minio": + if not os.path.exists(self.data_source_dir): + raise FileNotFoundError( + f"❌ 数据集目录不存在: {self.data_source_dir}\n" + f"\n配置信息:" + f"\n data_path = {data_path}" + f"\n dataset_name = {dataset_name}" + f"\n 拼接后路径 = {self.data_source_dir}" + f"\n\n请检查:" + f"\n 1. data_path 是否指向数据集的父目录" + f"\n 2. dataset_name 拼写是否正确" + f"\n 3. 数据集目录是否实际存在" + ) - # 验证必需的子目录 - required_subdirs = ["attributes", "timeseries"] - missing_dirs = [] - for subdir in required_subdirs: - subdir_path = os.path.join(self.data_source_dir, subdir) - if not os.path.exists(subdir_path): - missing_dirs.append(subdir) - - if missing_dirs: - raise FileNotFoundError( - f"❌ 数据集目录结构不完整: {self.data_source_dir}\n" - f"缺少以下子目录: {', '.join(missing_dirs)}" - ) + # 验证必需的子目录 + required_subdirs = ["attributes", "timeseries"] + missing_dirs = [] + for subdir in required_subdirs: + subdir_path = os.path.join(self.data_source_dir, subdir) + if not os.path.exists(subdir_path): + missing_dirs.append(subdir) + + if missing_dirs: + raise FileNotFoundError( + f"❌ 数据集目录结构不完整: {self.data_source_dir}\n" + f"缺少以下子目录: {', '.join(missing_dirs)}" + ) def get_constants(self): """ @@ -316,13 +317,13 @@ def _create_event_dict( if include_peak_obs: peak_flow = np.nanmax(inflow) if peak_flow < 1e-6: - return {} + return None event_dict["peak_obs"] = peak_flow return event_dict except Exception: - return {} + return None def load_1basin_flood_events( self, diff --git a/hydrodatasource/reader/grdc.py b/hydrodatasource/reader/grdc.py index da49308..9efbcda 100644 --- a/hydrodatasource/reader/grdc.py +++ b/hydrodatasource/reader/grdc.py @@ -30,6 +30,7 @@ class Grdc(HydroData): def __init__(self, data_path=None, *, uri=None): self.data_source_dir = str(uri) if uri is not None else data_path + self.dataset_name = Path(str(uri)).name if uri else os.path.basename(str(data_path)) self.data_source_description = self.set_data_source_describe() self.grdc_site_info = self.read_site_info() diff --git a/hydrodatasource/reader/rainfall_reader.py b/hydrodatasource/reader/rainfall_reader.py index df7edb0..3d7b427 100644 --- a/hydrodatasource/reader/rainfall_reader.py +++ b/hydrodatasource/reader/rainfall_reader.py @@ -9,6 +9,7 @@ import collections import os +from pathlib import Path import pandas as pd from tqdm import tqdm @@ -91,6 +92,7 @@ def __init__(self, data_folder=None, *, uri=None): _description_ """ self.data_source_dir = str(uri) if uri is not None else data_folder + self.dataset_name = Path(str(uri)).name if uri else os.path.basename(str(data_folder)) self.data_source_description = self.set_data_source_describe() self.station_info = self.read_station_info() diff --git a/hydrodatasource/reader/reservoir_datasets.py b/hydrodatasource/reader/reservoir_datasets.py index 6fad3dc..48b0fad 100644 --- a/hydrodatasource/reader/reservoir_datasets.py +++ b/hydrodatasource/reader/reservoir_datasets.py @@ -10,6 +10,7 @@ import os import collections +from pathlib import Path import geopandas as gpd from hydrodatasource.reader.data_source import HydroData @@ -19,6 +20,7 @@ class Crd(HydroData): def __init__(self, data_path=None, *, uri=None): self.data_source_dir = str(uri) if uri is not None else data_path + self.dataset_name = Path(str(uri)).name if uri else os.path.basename(str(data_path)) self.data_source_description = self.set_data_source_describe() self.reservoir_info = self.read_reservoir_info() diff --git a/hydrodatasource/reader/rsvr_inflow_reader.py b/hydrodatasource/reader/rsvr_inflow_reader.py index 311eb23..38024e6 100644 --- a/hydrodatasource/reader/rsvr_inflow_reader.py +++ b/hydrodatasource/reader/rsvr_inflow_reader.py @@ -10,6 +10,7 @@ import collections import os +from pathlib import Path import pandas as pd from tqdm import tqdm @@ -20,6 +21,7 @@ class RsvrInflowReader(HydroData): def __init__(self, data_folder=None, *, uri=None): self.data_source_dir = str(uri) if uri is not None else data_folder + self.dataset_name = Path(str(uri)).name if uri else os.path.basename(str(data_folder)) self.data_source_description = self.set_data_source_describe() self.rsvr_info = self.read_rsvr_info() diff --git a/tests/test_data_resolver.py b/tests/test_data_resolver.py index 3ade2c0..fa553f4 100644 --- a/tests/test_data_resolver.py +++ b/tests/test_data_resolver.py @@ -109,7 +109,7 @@ def test_resolve_local_path(self, temp_root, datasets_yml, monkeypatch): from hydrodatasource.configs.data_resolver import resolve_data_path monkeypatch.setattr( - "hydrodatasource.configs.data_resolver.get_local_root", + "hydrodataset.configs.data_resolver.get_local_root", lambda: temp_root / "data", ) @@ -127,7 +127,7 @@ def test_resolve_unknown_dataset_raises(self, datasets_yml, monkeypatch, tmp_pat ) monkeypatch.setattr( - "hydrodatasource.configs.data_resolver.get_local_root", + "hydrodataset.configs.data_resolver.get_local_root", lambda: tmp_path / "data", ) @@ -153,7 +153,7 @@ def test_resolve_cloud_s3_uri(self, datasets_yml, monkeypatch): from hydrodatasource.configs.data_resolver import resolve_data_path monkeypatch.setattr( - "hydrodatasource.configs.data_resolver.get_storage_config", + "hydrodataset.configs.data_resolver.get_storage_config", lambda: { "s3": { "bucket": "hydro-data", @@ -166,3 +166,18 @@ def test_resolve_cloud_s3_uri(self, datasets_yml, monkeypatch): "songliao_event", source="cloud", project_root=str(datasets_yml) ) assert uri == "s3://hydro-data/hydromodel/projects/songliao/event" + + def test_resolve_with_local_root_override(self, tmp_path): + """local_root overrides storage.local.root for project-level override.""" + from hydrodatasource.configs.data_resolver import resolve_data_path + + custom_root = tmp_path / "custom" + custom_root.mkdir(parents=True) + dataset_dir = custom_root / "projects" / "songliao" / "event" + dataset_dir.mkdir(parents=True) + + uri = resolve_data_path( + "songliao_event", source="local", local_root=str(custom_root) + ) + expected = str(dataset_dir) + assert uri == expected diff --git a/tests/test_reader_uri_constructors.py b/tests/test_reader_uri_constructors.py index 4d8711d..84bbc44 100644 --- a/tests/test_reader_uri_constructors.py +++ b/tests/test_reader_uri_constructors.py @@ -179,6 +179,41 @@ def test_rsvr_inflow_legacy_construction(self, mock_read_site_info): ds = RsvrInflowReader(data_folder="/data/rsvr_inflow") assert ds.data_source_dir == "/data/rsvr_inflow" + @pytest.mark.parametrize( + "reader_cls_factory, expected_name", + [ + ("grdc", "grdc"), + ("rainfall", "rainfall"), + ("crd", "crd"), + ("rsvrinflow", "rsvrinflow"), + ], + ) + def test_pattern_b_readers_have_dataset_name( + self, mock_read_site_info, reader_cls_factory, expected_name + ): + """Pattern B readers set dataset_name from URI (uri mode).""" + if reader_cls_factory == "grdc": + from hydrodatasource.reader.grdc import Grdc + + ds = Grdc(uri=f"/data/{expected_name}") + elif reader_cls_factory == "rainfall": + from hydrodatasource.reader.rainfall_reader import RainfallReader + + ds = RainfallReader(uri=f"/data/{expected_name}") + elif reader_cls_factory == "crd": + from hydrodatasource.reader.reservoir_datasets import Crd + + ds = Crd(uri=f"/data/{expected_name}") + elif reader_cls_factory == "rsvrinflow": + from hydrodatasource.reader.rsvr_inflow_reader import RsvrInflowReader + + ds = RsvrInflowReader(uri=f"/data/{expected_name}") + + assert hasattr(ds, "dataset_name"), f"{reader_cls_factory}: no dataset_name attr" + assert ds.dataset_name == expected_name, ( + f"{reader_cls_factory}: expected {expected_name}, got {ds.dataset_name}" + ) + # ── Pattern A classes (call super().__init__ via SelfMadeHydroDataset) ────── @@ -394,5 +429,37 @@ def test_legacy_construction_with_minimal_data(self, minimal_dataset): assert str(minimal_dataset) in ds.data_source_dir +class TestS3UriPathCheck: + """FloodEventDatasource with S3 uris should skip os.path.exists checks.""" + + def test_s3_uri_does_not_raise_filenotfound(self, monkeypatch): + """S3 URIs skip local filesystem checks — no FileNotFoundError.""" + from hydrodatasource.reader.floodevent import FloodEventDatasource + + # Prevent network calls triggered by set_data_source_describe + monkeypatch.setattr( + "hydrodatasource.reader.data_source.SelfMadeHydroDataset.set_data_source_describe", + lambda self: {"TS_DIRS": [], "ATTR_FILE": "", "UNIT_FILES": [], "SHAPE_DIR": ""}, + ) + monkeypatch.setattr( + "hydrodatasource.reader.data_source.SelfMadeHydroDataset.read_site_info", + lambda self: type("obj", (), {"__getitem__": lambda s, k: type("arr", (), {"values": []})()})(), + ) + + ds = FloodEventDatasource( + uri="s3://hydro-data/hydromodel/projects/songliao/event" + ) + assert ds.head == "minio" + assert "s3://" in ds.data_source_dir + + def test_local_nonexistent_path_raises(self, tmp_path): + """Non-existent local path still raises FileNotFoundError.""" + from hydrodatasource.reader.floodevent import FloodEventDatasource + + bad_path = str(tmp_path / "nonexistent" / "dataset") + with pytest.raises(FileNotFoundError): + FloodEventDatasource(uri=bad_path) + + # ── Import fix: needed for patch.object ───────────────────────────────────── from hydrodatasource.reader.data_source import SelfMadeHydroDataset From 8bcda80f4f7929006d8c0cd99fa4da2d9d090db2 Mon Sep 17 00:00:00 2001 From: owen Date: Tue, 23 Jun 2026 15:47:28 +0800 Subject: [PATCH 20/34] refactor: lazy-load config globals, unify CACHE_DIR with hydrodataset - config.py: replace import-time _init_settings() call with module __getattr__ lazy loading. SETTING/CACHE_DIR/LOCAL_DATA_PATH/FS/MINIO_PARAM are now initialized on first access, eliminating import-time I/O and S3 connection attempts. - CACHE_DIR now delegates to hydrodataset's get_cache_dir(), ensuring a single canonical cache path across repos. Removes the divergence where CACHE_DIR and get_cache_dir() returned different directories. - _init_settings() cleans stale __dict__ entries (left by monkeypatch teardowns) to keep __getattr__ resolution correct across test runs. - tests: add test_cache_dir_consistent_with_get_cache_dir --- hydrodatasource/configs/config.py | 72 +++++++++++++++++++------------ tests/test_config_api.py | 24 +++++++++++ 2 files changed, 68 insertions(+), 28 deletions(-) diff --git a/hydrodatasource/configs/config.py b/hydrodatasource/configs/config.py index b78608f..2491706 100644 --- a/hydrodatasource/configs/config.py +++ b/hydrodatasource/configs/config.py @@ -13,14 +13,14 @@ Database access is decoupled to a separate real-time service. Critical globals (SETTING, CACHE_DIR, FS, LOCAL_DATA_PATH, MINIO_PARAM) are -initialized at import time for backward compatibility. -New code should use the pure-function APIs (get_local_root, get_cache_dir). +lazy-loaded via module __getattr__ to avoid import-time side effects. FilePath: \\hydrodatasource\\hydrodatasource\\configs\\config.py Copyright (c) 2023-2026 Wenyu Ouyang. All rights reserved. """ import os +import sys from pathlib import Path from typing import Any, Dict, Optional @@ -51,18 +51,21 @@ def _load_settings_from_file() -> Dict[str, Any]: return setting -# ── Module-level state (initialized once at import for backward compat) ── +# ── Lazy-loading state ────────────────────────────────────────────────────── -SETTING: Dict[str, Any] = {} -CACHE_DIR: str = "" -LOCAL_DATA_PATH: str = "" -FS: Optional[s3fs.S3FileSystem] = None -MINIO_PARAM: Dict[str, str] = {} +_LAZY_KEYS = frozenset({"SETTING", "CACHE_DIR", "LOCAL_DATA_PATH", "FS", "MINIO_PARAM"}) +_lazy: Dict[str, Any] = {} +_initialized: bool = False def _init_settings() -> None: - """Load settings from ~/hydro_setting.yml once at module load time.""" - global SETTING, CACHE_DIR, LOCAL_DATA_PATH, FS, MINIO_PARAM + """Load settings from ~/hydro_setting.yml — called lazily on first access.""" + global _initialized + + # Remove stale __dict__ entries that may have been left by monkeypatch + # teardowns in tests (they shadow __getattr__ for lazy keys). + for key in _LAZY_KEYS: + globals().pop(key, None) try: setting = _load_settings_from_file() @@ -97,39 +100,50 @@ def _init_settings() -> None: } root = setting.get("local_data_path", {}).get("root", default_root) - cache = setting.get("local_data_path", {}).get( - "cache", os.path.join(Path.home(), "hydrodatasource_data", ".cache") - ) - SETTING = setting - LOCAL_DATA_PATH = root - CACHE_DIR = cache + _lazy["SETTING"] = setting + _lazy["LOCAL_DATA_PATH"] = root + _lazy["CACHE_DIR"] = str(_hd_get_cache_dir()) # Initialize S3FS only if minio credentials are present in old-format config minio_cfg = setting.get("minio", {}) if minio_cfg.get("client_endpoint") and minio_cfg.get("access_key"): - MINIO_PARAM = { + _lazy["MINIO_PARAM"] = { "endpoint_url": minio_cfg["client_endpoint"], "key": minio_cfg["access_key"], "secret": minio_cfg.get("secret", ""), } try: - FS = s3fs.S3FileSystem( - client_kwargs={"endpoint_url": MINIO_PARAM["endpoint_url"]}, - key=MINIO_PARAM["key"], - secret=MINIO_PARAM["secret"], + _lazy["FS"] = s3fs.S3FileSystem( + client_kwargs={"endpoint_url": _lazy["MINIO_PARAM"]["endpoint_url"]}, + key=_lazy["MINIO_PARAM"]["key"], + secret=_lazy["MINIO_PARAM"]["secret"], use_ssl=False, ) except Exception as e: print(f"Warning: S3FS initialization failed: {e}") - FS = None + _lazy["FS"] = None else: - MINIO_PARAM = {} - FS = None + _lazy["MINIO_PARAM"] = {} + _lazy["FS"] = None + + _initialized = True -# Initialize at module load (backward compat) -_init_settings() +def __getattr__(name: str) -> Any: + """Lazy-load critical globals on first access. + + Avoids import-time side effects (YAML reads, s3fs connections, print() + calls) — all deferred until a caller actually accesses SETTING, CACHE_DIR, + LOCAL_DATA_PATH, FS, or MINIO_PARAM. + """ + if name not in _LAZY_KEYS: + raise AttributeError( + f"module 'hydrodatasource.configs.config' has no attribute '{name}'" + ) + if not _initialized: + _init_settings() + return _lazy[name] # ── Public API (pure functions — no side effects) ───────────────────────── @@ -144,8 +158,10 @@ def get_local_root() -> Optional[Path]: root = _hd_get_local_root() if root is not None: return root - if LOCAL_DATA_PATH: - return Path(LOCAL_DATA_PATH) + # Use getattr to support monkeypatched values (tests) and __getattr__ fallback + local = getattr(sys.modules[__name__], "LOCAL_DATA_PATH", "") + if local: + return Path(local) return None diff --git a/tests/test_config_api.py b/tests/test_config_api.py index f1ffbc3..731bf38 100644 --- a/tests/test_config_api.py +++ b/tests/test_config_api.py @@ -285,3 +285,27 @@ def mock_load(): # Restore monkeypatch.setattr(config, "_load_settings_from_file", original_load) config._init_settings() + + def test_cache_dir_consistent_with_get_cache_dir(self, monkeypatch): + """CACHE_DIR delegates to hydrodataset get_cache_dir.""" + from hydrodatasource.configs import config + from pathlib import Path + + unified_cache = str(Path.home() / ".cache" / "unified_test") + + original_load = config._load_settings_from_file + + def mock_load(): + return {"storage": {"local": {"root": "/tmp/data"}}} + + monkeypatch.setattr(config, "_load_settings_from_file", mock_load) + monkeypatch.setattr(config, "_hd_get_cache_dir", lambda: Path(unified_cache)) + config._init_settings() + + assert config.CACHE_DIR == unified_cache, ( + f"CACHE_DIR={config.CACHE_DIR} != expected {unified_cache}" + ) + + # Restore + monkeypatch.setattr(config, "_load_settings_from_file", original_load) + config._init_settings() From 606983f96e8a6832549d0de9ae4b5740418448cf Mon Sep 17 00:00:00 2001 From: owen Date: Tue, 23 Jun 2026 18:33:51 +0800 Subject: [PATCH 21/34] =?UTF-8?q?refactor:=20unify=20data=20interface=20?= =?UTF-8?q?=E2=80=94=20URI-only=20constructors=20across=20all=2011=20reade?= =?UTF-8?q?r=20classes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replace (data_path, dataset_name) with single uri parameter in HydroData base - All 11 reader classes now accept uri (positional for Pattern B, keyword for Pattern A) - Remove legacy kwargs (data_path, dataset_name, data_folder) from all constructors - Add S3 URI guard: skip os.path.exists() when head='minio' in FloodEventDatasource - Add dataset_name auto-derivation from URI basename in HydroData.__init__ - Lazy-load config globals: rename LOCAL_DATA_PATH to LOCAL_ROOT, add warnings - Add data_resolver.py thin wrapper over hydrodataset public API with extra_registry_dicts - Add datasets.yml registry with songliao_event → floodevent mapping - Export READER_ALIASES, resolve_data_path, DatasetResolutionError from package - Add 74 tests covering URI constructors, config API, data resolver, and package exports - Align CACHE_DIR with hydrodataset defaults Closes #13 --- hydrodatasource/configs/config.py | 86 ++++------ hydrodatasource/reader/access_fs.py | 16 +- hydrodatasource/reader/data_source.py | 110 ++++--------- hydrodatasource/reader/floodevent.py | 37 ++--- hydrodatasource/reader/gages.py | 6 +- hydrodatasource/reader/grdc.py | 9 +- hydrodatasource/reader/rainfall_reader.py | 12 +- hydrodatasource/reader/reservoir_datasets.py | 5 +- hydrodatasource/reader/rsvr_inflow_reader.py | 5 +- pyproject.toml | 9 +- scripts/const4scripts.py | 5 +- tests/test_config_api.py | 163 ++++++++++--------- tests/test_data_source.py | 84 ++++------ tests/test_dmca_esr.py | 12 +- tests/test_gages.py | 12 +- tests/test_reader_uri_constructors.py | 121 +++++--------- tests/test_tg_hydro_datasource.py | 2 +- uv.lock | 34 +++- 18 files changed, 319 insertions(+), 409 deletions(-) diff --git a/hydrodatasource/configs/config.py b/hydrodatasource/configs/config.py index 2491706..6282ae1 100644 --- a/hydrodatasource/configs/config.py +++ b/hydrodatasource/configs/config.py @@ -12,7 +12,7 @@ Cloud access is unified under storage.s3. Database access is decoupled to a separate real-time service. -Critical globals (SETTING, CACHE_DIR, FS, LOCAL_DATA_PATH, MINIO_PARAM) are +Critical globals (SETTING, CACHE_DIR, FS, LOCAL_ROOT, MINIO_PARAM) are lazy-loaded via module __getattr__ to avoid import-time side effects. FilePath: \\hydrodatasource\\hydrodatasource\\configs\\config.py @@ -21,6 +21,7 @@ import os import sys +import warnings from pathlib import Path from typing import Any, Dict, Optional @@ -53,7 +54,7 @@ def _load_settings_from_file() -> Dict[str, Any]: # ── Lazy-loading state ────────────────────────────────────────────────────── -_LAZY_KEYS = frozenset({"SETTING", "CACHE_DIR", "LOCAL_DATA_PATH", "FS", "MINIO_PARAM"}) +_LAZY_KEYS = frozenset({"SETTING", "CACHE_DIR", "LOCAL_ROOT", "FS", "MINIO_PARAM"}) _lazy: Dict[str, Any] = {} _initialized: bool = False @@ -70,48 +71,30 @@ def _init_settings() -> None: try: setting = _load_settings_from_file() except (ValueError, FileNotFoundError) as e: - print(f"Warning: Could not load hydro_setting.yml: {e}") + warnings.warn(f"Could not load hydro_setting.yml: {e}", stacklevel=2) setting = {} default_root = os.path.join(Path.home(), "hydrodatasource_data") if not setting: - print(f"Using default data paths in home directory: {default_root}") - setting = { - "local_data_path": { - "root": default_root, - "datasets-origin": os.path.join(default_root, "datasets-origin"), - "datasets-interim": os.path.join(default_root, "datasets-interim"), - "cache": os.path.join(default_root, ".cache"), - } - } - - # Bridge: if new format (storage.*) exists but old (local_data_path) is - # missing, create derived local_data_path for backward compat. - if "local_data_path" not in setting and "storage" in setting: - storage_local = setting["storage"].get("local", {}) - storage_root = storage_local.get("root", default_root) - setting["local_data_path"] = { - "root": storage_root, - "datasets-origin": os.path.join(storage_root, "datasets-origin"), - "datasets-interim": os.path.join(storage_root, "datasets-interim"), - "cache": storage_local.get( - "cache", os.path.join(storage_root, ".cache") - ), - } + warnings.warn( + f"Using default data paths in home directory: {default_root}", + stacklevel=2, + ) + setting = {"storage": {"local": {"root": default_root}}} - root = setting.get("local_data_path", {}).get("root", default_root) + root = setting.get("storage", {}).get("local", {}).get("root", default_root) _lazy["SETTING"] = setting - _lazy["LOCAL_DATA_PATH"] = root + _lazy["LOCAL_ROOT"] = root _lazy["CACHE_DIR"] = str(_hd_get_cache_dir()) - # Initialize S3FS only if minio credentials are present in old-format config - minio_cfg = setting.get("minio", {}) - if minio_cfg.get("client_endpoint") and minio_cfg.get("access_key"): + # Initialize S3FS from storage.s3 credentials + s3_cfg = setting.get("storage", {}).get("s3", {}) + if s3_cfg.get("endpoint_url") and s3_cfg.get("key"): _lazy["MINIO_PARAM"] = { - "endpoint_url": minio_cfg["client_endpoint"], - "key": minio_cfg["access_key"], - "secret": minio_cfg.get("secret", ""), + "endpoint_url": s3_cfg["endpoint_url"], + "key": s3_cfg["key"], + "secret": s3_cfg.get("secret", ""), } try: _lazy["FS"] = s3fs.S3FileSystem( @@ -121,7 +104,7 @@ def _init_settings() -> None: use_ssl=False, ) except Exception as e: - print(f"Warning: S3FS initialization failed: {e}") + warnings.warn(f"S3FS initialization failed: {e}", stacklevel=2) _lazy["FS"] = None else: _lazy["MINIO_PARAM"] = {} @@ -135,7 +118,7 @@ def __getattr__(name: str) -> Any: Avoids import-time side effects (YAML reads, s3fs connections, print() calls) — all deferred until a caller actually accesses SETTING, CACHE_DIR, - LOCAL_DATA_PATH, FS, or MINIO_PARAM. + LOCAL_ROOT, FS, or MINIO_PARAM. """ if name not in _LAZY_KEYS: raise AttributeError( @@ -152,14 +135,13 @@ def __getattr__(name: str) -> Any: def get_local_root() -> Optional[Path]: """Get the local storage root directory. - Uses the new storage.local.root format (via hydrodataset settings). - Falls back to the old local_data_path.root format for backward compat. + Returns storage.local.root via hydrodataset settings. + Falls back to LOCAL_ROOT if set. """ root = _hd_get_local_root() if root is not None: return root - # Use getattr to support monkeypatched values (tests) and __getattr__ fallback - local = getattr(sys.modules[__name__], "LOCAL_DATA_PATH", "") + local = getattr(sys.modules[__name__], "LOCAL_ROOT", "") if local: return Path(local) return None @@ -178,8 +160,7 @@ def get_storage_config() -> Dict[str, Any]: def read_setting(setting_path: str) -> Dict[str, Any]: """Read and validate a hydro_setting.yml file. - Accepts both old (local_data_path.*) and new (storage.*) formats. - No longer requires minio or postgres sections. + Only the storage.* format is accepted. Old local_data_path.* format is rejected. """ if not os.path.exists(setting_path): raise FileNotFoundError(f"Configuration file not found: {setting_path}") @@ -188,22 +169,18 @@ def read_setting(setting_path: str) -> Dict[str, Any]: setting = yaml.safe_load(file) example_setting = ( - "# New format (recommended):\n" + "# Required format:\n" "storage:\n" " default_source: local\n" " local:\n" " root: 'D:\\\\data\\\\hydrodatasource'\n" + " cache: data\\\\cache\n" " s3:\n" + " endpoint_url: 'http://minio:9000'\n" + " key: 'access_key'\n" + " secret: 'secret_key'\n" " bucket: hydro-data\n" " prefix: hydromodel\n" - " region: us-east-1\n" - " profile: default\n\n" - "# Old format (deprecated, still supported):\n" - "local_data_path:\n" - " root: 'D:\\\\data\\\\waterism'\n" - " datasets-origin: 'D:\\\\data\\\\waterism\\\\datasets-origin'\n" - " datasets-interim: 'D:\\\\data\\\\waterism\\\\datasets-interim'\n" - " cache: 'D:\\\\data\\\\waterism\\\\.cache'\n" ) if setting is None: @@ -212,12 +189,9 @@ def read_setting(setting_path: str) -> Dict[str, Any]: f"Example configuration:\n{example_setting}" ) - has_old = "local_data_path" in setting - has_new = "storage" in setting - if not has_old and not has_new: + if "storage" not in setting: raise ValueError( - f"Configuration must have 'storage' (new format) or " - f"'local_data_path' (old format) section.\n\n" + f"Configuration must have 'storage' section.\n\n" f"Example configuration:\n{example_setting}" ) diff --git a/hydrodatasource/reader/access_fs.py b/hydrodatasource/reader/access_fs.py index cf60230..d35ecd2 100644 --- a/hydrodatasource/reader/access_fs.py +++ b/hydrodatasource/reader/access_fs.py @@ -52,7 +52,7 @@ def spec_path(url_path: str, head="local", need_cache=False, is_dir=False): else: ret_data = [] if head == "local": - url_path = os.path.join(conf.LOCAL_DATA_PATH, url_path) + url_path = os.path.join(conf.LOCAL_ROOT, url_path) ret_data.extend( read_valid_data(file, need_cache=need_cache) for file in glob.glob(url_path + "/**", recursive=True) @@ -101,19 +101,19 @@ def read_valid_data(obj: str, storage_option=None, need_cache=False, need_refer= if not dot_in_obj: data_obj = pd.read_fwf(obj, storage_options=storage_option) if (need_cache is True) & (storage_option is not None): - data_obj.to_file(path=os.path.join(conf.LOCAL_DATA_PATH, cache_name)) + data_obj.to_file(path=os.path.join(conf.LOCAL_ROOT, cache_name)) else: ext_name = obj.split(".")[-1] if ext_name == "csv": data_obj = pd.read_csv(obj, storage_options=storage_option) if need_cache & (storage_option is not None): data_obj.to_csv( - path_or_buf=os.path.join(conf.LOCAL_DATA_PATH, cache_name) + path_or_buf=os.path.join(conf.LOCAL_ROOT, cache_name) ) elif ext_name == "txt": data_obj = pd.read_fwf(obj, storage_options=storage_option) if need_cache & (storage_option is not None): - data_obj.to_csv(os.path.join(conf.LOCAL_DATA_PATH, cache_name)) + data_obj.to_csv(os.path.join(conf.LOCAL_ROOT, cache_name)) elif (ext_name in ["nc", "nc4", "h5", "hdf5"]) or ("nc4" in obj): if need_refer: data_obj = gen_refer_and_read_zarr(obj, storage_option=storage_option) @@ -140,12 +140,12 @@ def read_valid_data(obj: str, storage_option=None, need_cache=False, need_refer= phony_dims="access", ) if (need_cache is True) & (storage_option is not None): - data_obj.to_netcdf(path=os.path.join(conf.LOCAL_DATA_PATH, cache_name)) + data_obj.to_netcdf(path=os.path.join(conf.LOCAL_ROOT, cache_name)) elif ext_name == "json": data_obj = pd.read_json(obj, storage_options=storage_option) if (need_cache is True) & (storage_option is not None): data_obj.to_json( - path_or_buf=os.path.join(conf.LOCAL_DATA_PATH, cache_name) + path_or_buf=os.path.join(conf.LOCAL_ROOT, cache_name) ) elif ext_name == "zip": # Now zipfile is used to read shapefile @@ -156,7 +156,7 @@ def read_valid_data(obj: str, storage_option=None, need_cache=False, need_refer= else: data_obj = gpd.read_file(obj) if (need_cache is True) & (storage_option is not None): - data_obj.to_file(path=os.path.join(conf.LOCAL_DATA_PATH, cache_name)) + data_obj.to_file(path=os.path.join(conf.LOCAL_ROOT, cache_name)) elif "grb2" in obj: if storage_option is not None: obj = f"simplecache::{obj}" @@ -171,7 +171,7 @@ def read_valid_data(obj: str, storage_option=None, need_cache=False, need_refer= else: grib_ds = xr.open_dataset(obj) if (need_cache is True) & (storage_option is not None): - grib_ds.to_netcdf(path=os.path.join(conf.LOCAL_DATA_PATH, cache_name)) + grib_ds.to_netcdf(path=os.path.join(conf.LOCAL_ROOT, cache_name)) elif ext_name == "zarr": if storage_option is not None: zarr_mapper = conf.FS.get_mapper(obj) diff --git a/hydrodatasource/reader/data_source.py b/hydrodatasource/reader/data_source.py index 584c466..d22d6e6 100644 --- a/hydrodatasource/reader/data_source.py +++ b/hydrodatasource/reader/data_source.py @@ -33,30 +33,19 @@ class HydroData(ABC): - """An interface for reading multi-modal data sources. + """Abstract base class for reading multi-modal hydrological data sources. - Supports two construction patterns: - 1. New (resolved URI): HydroData(uri="/path/to/data") - 2. Legacy: HydroData(data_path="/parent", dataset_name="child") + Single construction pattern: HydroData(uri). Parameters ---------- - data_path : str, optional - Parent directory path (legacy mode). - dataset_name : str, optional - Subdirectory name under data_path (legacy mode). - uri : str, optional - Resolved absolute URI pointing directly to the data directory. - When provided, data_path and dataset_name are ignored. + uri : str or Path + Absolute path or S3 URI pointing directly to the data directory. """ - def __init__(self, data_path=None, dataset_name=None, *, uri=None): - if uri is not None: - self.data_source_dir = str(uri) - self.dataset_name = dataset_name or Path(str(uri)).name - else: - self.data_source_dir = os.path.join(data_path, dataset_name) - self.dataset_name = dataset_name + def __init__(self, uri): + self.data_source_dir = str(uri) + self.dataset_name = Path(str(uri)).name def get_name(self): raise NotImplementedError @@ -80,26 +69,18 @@ class SelfMadeHydroDataset(HydroData): def __init__( self, - data_path=None, - dataset_name=None, - time_unit=None, - *, uri=None, + time_unit=None, **kwargs, ): """Initialize a self-made Caravan-style dataset. Parameters ---------- - data_path : str, optional - Parent directory path (legacy mode). - dataset_name : str, optional - Dataset name (legacy mode, or label when uri is used). + uri : str, optional + Absolute URI pointing directly to the data directory. time_unit : list, optional Time units to process, by default None. - uri : str, optional - Resolved absolute URI pointing directly to the data directory. - When provided, data_path and dataset_name are used as labels only. kwargs : dict, optional Additional keyword arguments, by default None. """ @@ -110,13 +91,9 @@ def __init__( "time_unit must be one of ['1h', '3h', '1D', '8D','1M']. We only support these time units now." ) # Determine filesystem head from URI or data_path - effective_path = str(uri) if uri is not None else (data_path or "") + effective_path = str(uri) if uri is not None else "" self.head = "minio" if "s3://" in effective_path else "local" - super().__init__( - data_path=data_path, - dataset_name=dataset_name, - uri=uri, - ) + super().__init__(uri=uri) self.data_source_description = self.set_data_source_describe() self.camels_sites = self.read_site_info() self.time_unit = time_unit @@ -883,16 +860,12 @@ def read_mean_prcp(self, gage_id_lst=None, unit="mm/d"): class LongTermDataset(SelfMadeHydroDataset): - def __init__(self, data_path=None, download=False, time_unit=None, *, uri=None, **kwargs): + def __init__(self, uri=None, download=False, time_unit=None, **kwargs): if time_unit is None: time_unit = ["1M"] - if "dataset_name" not in kwargs: - kwargs["dataset_name"] = "gmspa" super().__init__( - data_path=data_path, - dataset_name=kwargs.pop("dataset_name", str(download)), - time_unit=time_unit, uri=uri, + time_unit=time_unit, **kwargs, ) @@ -949,23 +922,20 @@ def cache_global_dataset(self, object_ids: list = None, t_range_list: list = Non class SelfMadeForecastDataset(SelfMadeHydroDataset): """For selfmadehydrodataset, we design a new file format for forecast data from GFS et al.""" - def __init__(self, data_path=None, dataset_name=None, time_unit=None, *, uri=None): - """intialize a Class for reading forecast data + def __init__(self, uri=None, time_unit=None, **kwargs): + """Initialize a class for reading forecast data. Parameters ---------- - data_path : str - the path of data source + uri : str + Absolute path or S3 URI pointing to the forecast data directory. time_unit : list, optional - unit of one time period, by default None - dataset_name: str - name will be used for cache files + Unit of one time period, by default None. """ super().__init__( - data_path=data_path, - dataset_name=dataset_name, - time_unit=time_unit, uri=uri, + time_unit=time_unit, + **kwargs, ) def set_data_source_describe(self): @@ -1363,25 +1333,21 @@ class StationHydroDataset(SelfMadeHydroDataset): - adjacency_xxx_True.csv """ - def __init__(self, data_path=None, dataset_name=None, time_unit=None, *, uri=None, **kwargs): + def __init__(self, uri=None, time_unit=None, **kwargs): """Initialize StationHydroDataset. Parameters ---------- - data_path : str - Path to the dataset directory - dataset_name : str - Name of the dataset + uri : str + Absolute path or S3 URI pointing to the station dataset directory. time_unit : list, optional Time units for the data, by default None **kwargs : dict Additional keyword arguments passed to parent class """ super().__init__( - data_path=data_path, - dataset_name=dataset_name, - time_unit=time_unit, uri=uri, + time_unit=time_unit, **kwargs, ) self.station_info = None @@ -1967,33 +1933,21 @@ class TgHydroDatasource(SelfMadeHydroDataset): └── shapes/ # 区间流域形状文件 """ - def __init__(self, data_path=None, dataset_name=None, time_unit=None, *, uri=None, **kwargs): - """ - 初始化TG流域数据集 + def __init__(self, uri=None, time_unit=None, **kwargs): + """Initialize TG basin dataset. Parameters ---------- - data_path : str - 数据根目录路径 - dataset_name : str, optional - 数据集名称, by default None + uri : str + Absolute path or S3 URI pointing to the TG dataset directory. time_unit : list, optional - 时间单位列表, by default None + Time unit list, by default None. **kwargs : dict - 其他参数 + Other parameters. """ - # # 调用父类初始化 - # self.inter_basin_pred_file = kwargs.get("inter_basin_pred_file", None) - # if self.inter_basin_pred_file is None: - # raise ValueError( - # "inter_basin_pred_file is required; please run scripts/generate_inter_basin_predictions.py to generate the inter-basin predictions." - # ) - # kwargs.pop("inter_basin_pred_file") super().__init__( - data_path=data_path, - dataset_name=dataset_name, - time_unit=time_unit, uri=uri, + time_unit=time_unit, **kwargs, ) diff --git a/hydrodatasource/reader/floodevent.py b/hydrodatasource/reader/floodevent.py index c3ef5e4..2d7ce47 100644 --- a/hydrodatasource/reader/floodevent.py +++ b/hydrodatasource/reader/floodevent.py @@ -32,11 +32,9 @@ class FloodEventDatasource(SelfMadeHydroDataset): def __init__( self, - data_path: str = None, - dataset_name: str = "songliaorrevents", + uri: str = None, time_unit: Optional[List[str]] = None, *, - uri: str = None, rain_key: str = "rain", pet_key: str = "ES", net_rain_key: str = "net_rain", @@ -49,10 +47,8 @@ def __init__( Parameters ---------- - data_path : str - Path to the data. - dataset_name : str, optional - Name of the dataset. + uri : str + Absolute path or S3 URI pointing to the data directory. time_unit : list of str, optional List of time units, default is ["3h"]. rain_key : str, optional @@ -71,15 +67,14 @@ def __init__( time_unit = ["3h"] # Derive delta_t_hours from time_unit - # 从time_unit推导出时间步长（小时） - primary_time_unit = time_unit[0] # 使用第一个时间单位作为主要单位 + primary_time_unit = time_unit[0] delta_t_hours = FloodEventDatasource._parse_time_unit_to_hours( primary_time_unit ) # Store constants as instance attributes self.rain_key = rain_key - self.pet_key = pet_key # 假设数据集中蒸散发的列名为 'ES' + self.pet_key = pet_key self.net_rain_key = net_rain_key self.obs_flow_key = obs_flow_key self.warmup_length = warmup_length @@ -87,29 +82,19 @@ def __init__( self.delta_t_seconds = self.delta_t_hours * 3600.0 super().__init__( - data_path=data_path, - time_unit=time_unit, - dataset_name=dataset_name, uri=uri, + time_unit=time_unit, **kwargs, ) - # 验证数据目录是否存在（S3/云端路径跳过本地文件系统检查） + # Skip local filesystem checks for S3/cloud paths if self.head != "minio": if not os.path.exists(self.data_source_dir): raise FileNotFoundError( - f"❌ 数据集目录不存在: {self.data_source_dir}\n" - f"\n配置信息:" - f"\n data_path = {data_path}" - f"\n dataset_name = {dataset_name}" - f"\n 拼接后路径 = {self.data_source_dir}" - f"\n\n请检查:" - f"\n 1. data_path 是否指向数据集的父目录" - f"\n 2. dataset_name 拼写是否正确" - f"\n 3. 数据集目录是否实际存在" + f"Dataset directory does not exist: {self.data_source_dir}\n" + f"uri = {uri}" ) - # 验证必需的子目录 required_subdirs = ["attributes", "timeseries"] missing_dirs = [] for subdir in required_subdirs: @@ -119,8 +104,8 @@ def __init__( if missing_dirs: raise FileNotFoundError( - f"❌ 数据集目录结构不完整: {self.data_source_dir}\n" - f"缺少以下子目录: {', '.join(missing_dirs)}" + f"Dataset directory structure incomplete: {self.data_source_dir}\n" + f"Missing subdirectories: {', '.join(missing_dirs)}" ) def get_constants(self): diff --git a/hydrodatasource/reader/gages.py b/hydrodatasource/reader/gages.py index 69badc8..d5faaee 100644 --- a/hydrodatasource/reader/gages.py +++ b/hydrodatasource/reader/gages.py @@ -21,12 +21,14 @@ from pandas.core.dtypes.common import is_string_dtype, is_numeric_dtype from tqdm import tqdm from hydroutils import hydro_logger, hydro_file, hydro_stat +import hydrodatasource.configs.config as conf +from hydrodatasource.configs.config import CACHE_DIR from hydrodatasource.reader.data_source import HydroData class Gages(HydroData): - def __init__(self, data_path=None, dataset_name=None, *, uri=None): - super().__init__(data_path=data_path, dataset_name=dataset_name, uri=uri) + def __init__(self, uri): + super().__init__(uri) self.data_source_description = self.set_data_source_describe() self.gages_sites = self.read_site_info() diff --git a/hydrodatasource/reader/grdc.py b/hydrodatasource/reader/grdc.py index 9efbcda..0a8d392 100644 --- a/hydrodatasource/reader/grdc.py +++ b/hydrodatasource/reader/grdc.py @@ -21,16 +21,15 @@ import geopandas as gpd from dateutil.parser import parse -from hydrodatasource.configs.config import CACHE_DIR, SETTING +from hydrodatasource.configs.config import CACHE_DIR, get_local_root from hydrodatasource.reader.data_source import HydroData class Grdc(HydroData): """Reading GRDC streamflow data.""" - def __init__(self, data_path=None, *, uri=None): - self.data_source_dir = str(uri) if uri is not None else data_path - self.dataset_name = Path(str(uri)).name if uri else os.path.basename(str(data_path)) + def __init__(self, uri): + super().__init__(uri) self.data_source_description = self.set_data_source_describe() self.grdc_site_info = self.read_site_info() @@ -610,7 +609,7 @@ def _convert_to_serializable(obj): if __name__ == "__main__": - data_dir = os.path.join(SETTING["local_data_path"]["datasets-origin"], "GRDC") + data_dir = str(get_local_root() / "datasets-origin" / "GRDC") grdc = Grdc(data_dir) # grdc.cache_grdc_daily(["1107700", "4101200"], ["1990-10-01", "2000-10-01"]) grdc.cache_grdc_daily() diff --git a/hydrodatasource/reader/rainfall_reader.py b/hydrodatasource/reader/rainfall_reader.py index 3d7b427..54fcee2 100644 --- a/hydrodatasource/reader/rainfall_reader.py +++ b/hydrodatasource/reader/rainfall_reader.py @@ -82,17 +82,15 @@ def filter_out_rows(df1, df2, key_columns): @hydro_logger class RainfallReader(HydroData): - def __init__(self, data_folder=None, *, uri=None): - """A reader for basin rainfall data - Even for station data, we still use basin as the index + def __init__(self, uri): + """A reader for basin rainfall data. Parameters ---------- - data_folder : _type_ - _description_ + uri : str + Absolute path pointing to the rainfall data directory. """ - self.data_source_dir = str(uri) if uri is not None else data_folder - self.dataset_name = Path(str(uri)).name if uri else os.path.basename(str(data_folder)) + super().__init__(uri) self.data_source_description = self.set_data_source_describe() self.station_info = self.read_station_info() diff --git a/hydrodatasource/reader/reservoir_datasets.py b/hydrodatasource/reader/reservoir_datasets.py index 48b0fad..b2043e0 100644 --- a/hydrodatasource/reader/reservoir_datasets.py +++ b/hydrodatasource/reader/reservoir_datasets.py @@ -18,9 +18,8 @@ class Crd(HydroData): """Reading CRD reservoir dataset""" - def __init__(self, data_path=None, *, uri=None): - self.data_source_dir = str(uri) if uri is not None else data_path - self.dataset_name = Path(str(uri)).name if uri else os.path.basename(str(data_path)) + def __init__(self, uri): + super().__init__(uri) self.data_source_description = self.set_data_source_describe() self.reservoir_info = self.read_reservoir_info() diff --git a/hydrodatasource/reader/rsvr_inflow_reader.py b/hydrodatasource/reader/rsvr_inflow_reader.py index 38024e6..06c4a88 100644 --- a/hydrodatasource/reader/rsvr_inflow_reader.py +++ b/hydrodatasource/reader/rsvr_inflow_reader.py @@ -19,9 +19,8 @@ class RsvrInflowReader(HydroData): - def __init__(self, data_folder=None, *, uri=None): - self.data_source_dir = str(uri) if uri is not None else data_folder - self.dataset_name = Path(str(uri)).name if uri else os.path.basename(str(data_folder)) + def __init__(self, uri): + super().__init__(uri) self.data_source_description = self.set_data_source_describe() self.rsvr_info = self.read_rsvr_info() diff --git a/pyproject.toml b/pyproject.toml index cdfe6bd..5d6e1af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,6 +61,7 @@ dev = [ "pytest-runner", "pytest-asyncio", "pytest-mock", + "pytest-dotenv", # Packaging "wheel", @@ -138,8 +139,7 @@ testpaths = ["tests"] python_files = ["test_*.py"] python_classes = ["Test*"] python_functions = ["test_*"] -collect_ignore = ["setup.py"] - +env_files = [".env"] [tool.black] line-length = 88 target-version = ['py310', 'py311', 'py312'] @@ -178,6 +178,11 @@ exclude = [ [tool.uv.sources] hydrodataset = { path = "../hydrodataset", editable = true } +[dependency-groups] +dev = [ + "pytest-dotenv>=0.5.2", +] + [tool.uv] # Override aqua-fetch's outdated dependency constraints # aqua-fetch 限制了 pandas<=2.1.4，但实际上新版本的 pandas 和 xarray 已经修复了兼容性问题 diff --git a/scripts/const4scripts.py b/scripts/const4scripts.py index 92b6121..7738fe8 100644 --- a/scripts/const4scripts.py +++ b/scripts/const4scripts.py @@ -13,7 +13,7 @@ # and modify the paths as your own paths in definitions_private.py import os -from hydrodataset import SETTING +from hydrodataset.configs.settings import get_local_root try: import const4scripts_private @@ -27,4 +27,5 @@ # where to put results RESULT_DIR = r"C:\Users\wenyu\code\hydrodatasource\results" # where are the data sources - DATASET_DIR = SETTING["local_data_path"]["basins-origin"] + root = get_local_root() + DATASET_DIR = os.path.join(str(root), "basins-origin") if root else os.path.join(os.getcwd(), "data") diff --git a/tests/test_config_api.py b/tests/test_config_api.py index 731bf38..a49ad96 100644 --- a/tests/test_config_api.py +++ b/tests/test_config_api.py @@ -4,12 +4,14 @@ and read_setting() validation, plus backward-compatible globals. These tests verify the changes made in feat/unified-data-interface. + +Only the new storage.* config format is supported. Old local_data_path.* +and minio.* formats have been removed. """ import os import tempfile from pathlib import Path -from unittest.mock import patch import pytest import yaml @@ -19,7 +21,7 @@ class TestReadSetting: - """Verify read_setting() accepts both old and new config formats.""" + """Verify read_setting() accepts the new storage.* config format only.""" def test_new_format_storage_section(self, tmp_path): """New format with storage.* sections is accepted.""" @@ -33,6 +35,9 @@ def test_new_format_storage_section(self, tmp_path): "default_source": "local", "local": {"root": str(tmp_path / "data")}, "s3": { + "endpoint_url": "http://minio:9000", + "key": "mykey", + "secret": "mysecret", "bucket": "my-bucket", "prefix": "hydromodel", }, @@ -46,8 +51,8 @@ def test_new_format_storage_section(self, tmp_path): assert "storage" in result assert result["storage"]["s3"]["bucket"] == "my-bucket" - def test_old_format_local_data_path(self, tmp_path): - """Old format with local_data_path.* is accepted (backward compat).""" + def test_old_format_rejected(self, tmp_path): + """Old format (local_data_path.*) raises ValueError.""" from hydrodatasource.configs.config import read_setting setting_path = tmp_path / "hydro_setting.yml" @@ -57,38 +62,14 @@ def test_old_format_local_data_path(self, tmp_path): "local_data_path": { "root": str(tmp_path / "data"), "datasets-origin": str(tmp_path / "data" / "datasets-origin"), - "datasets-interim": str(tmp_path / "data" / "datasets-interim"), } } ), encoding="utf-8", ) - result = read_setting(str(setting_path)) - assert "local_data_path" in result - - def test_both_formats_accepted(self, tmp_path): - """Config with both old and new sections is valid.""" - from hydrodatasource.configs.config import read_setting - - setting_path = tmp_path / "hydro_setting.yml" - setting_path.write_text( - yaml.dump( - { - "storage": { - "local": {"root": str(tmp_path / "data")}, - }, - "local_data_path": { - "root": str(tmp_path / "data"), - }, - } - ), - encoding="utf-8", - ) - - result = read_setting(str(setting_path)) - assert "storage" in result - assert "local_data_path" in result + with pytest.raises(ValueError, match="must have 'storage'"): + read_setting(str(setting_path)) def test_missing_file_raises(self, tmp_path): """Missing setting file raises FileNotFoundError.""" @@ -107,8 +88,8 @@ def test_empty_setting_raises(self, tmp_path): with pytest.raises(ValueError, match="empty"): read_setting(str(setting_path)) - def test_missing_required_sections_raises(self, tmp_path): - """Setting without 'storage' or 'local_data_path' raises ValueError.""" + def test_missing_storage_section_raises(self, tmp_path): + """Setting without 'storage' section raises ValueError.""" from hydrodatasource.configs.config import read_setting setting_path = tmp_path / "hydro_setting.yml" @@ -126,35 +107,25 @@ def test_missing_required_sections_raises(self, tmp_path): class TestGetLocalRoot: """Verify get_local_root() resolves correctly.""" - def test_returns_path_when_local_data_path_set(self, monkeypatch): - """get_local_root returns LOCAL_DATA_PATH when not None.""" - from hydrodatasource.configs import config - - monkeypatch.setattr(config, "LOCAL_DATA_PATH", "/test/root") - # Patch _hd_get_local_root to return None (no new format) - monkeypatch.setattr(config, "_hd_get_local_root", lambda: None) - - result = config.get_local_root() - assert result == Path("/test/root") - - def test_returns_hd_root_when_available(self, monkeypatch): - """get_local_root prefers hydrodataset's root when available.""" + def test_returns_path_from_storage_local_root(self, monkeypatch): + """get_local_root returns storage.local.root from settings.""" from hydrodatasource.configs import config + from pathlib import Path monkeypatch.setattr( - config, "_hd_get_local_root", lambda: Path("/hd/root") + config, "_hd_get_local_root", lambda: Path("/storage/local/root") ) - monkeypatch.setattr(config, "LOCAL_DATA_PATH", "/old/root") + monkeypatch.setattr(config, "LOCAL_ROOT", "") result = config.get_local_root() - assert result == Path("/hd/root") + assert result == Path("/storage/local/root") def test_returns_none_when_nothing_configured(self, monkeypatch): """get_local_root returns None when nothing is configured.""" from hydrodatasource.configs import config monkeypatch.setattr(config, "_hd_get_local_root", lambda: None) - monkeypatch.setattr(config, "LOCAL_DATA_PATH", "") + monkeypatch.setattr(config, "LOCAL_ROOT", "") result = config.get_local_root() assert result is None @@ -205,11 +176,11 @@ def test_setting_is_dict(self): assert isinstance(SETTING, dict) - def test_local_data_path_is_str(self): - """LOCAL_DATA_PATH is a str (may be empty).""" - from hydrodatasource.configs.config import LOCAL_DATA_PATH + def test_local_root_is_str(self): + """LOCAL_ROOT is a str (may be empty).""" + from hydrodatasource.configs.config import LOCAL_ROOT - assert isinstance(LOCAL_DATA_PATH, str) + assert isinstance(LOCAL_ROOT, str) def test_cache_dir_is_str(self): """CACHE_DIR is a str (may be empty).""" @@ -224,25 +195,18 @@ def test_minio_param_is_dict(self): assert isinstance(MINIO_PARAM, dict) -# ── Format bridging tests ─────────────────────────────────────────────────── +# ── Format bridging tests (new format only, bridge removed) ──────────────── class TestFormatBridging: - """Verify old-format and new-format interop in _init_settings.""" + """Verify new storage.* format initializes correctly.""" - def test_new_format_derives_local_data_path(self, monkeypatch, tmp_path): - """When only storage.* is set, local_data_path is derived.""" + def test_no_local_data_path_in_setting(self, monkeypatch, tmp_path): + """SETTING should NOT contain local_data_path key (bridge removed).""" from hydrodatasource.configs import config - import importlib - # Create a temp setting file with only new format - setting_path = Path.home() / "hydro_setting.yml" - # We can't modify the real home config, so test the logic directly - # by calling _init_settings with mocked _load_settings_from_file data_root = tmp_path / "data" - original_load = config._load_settings_from_file - def mock_load(): return { "storage": { @@ -251,28 +215,32 @@ def mock_load(): } } + original_load = config._load_settings_from_file monkeypatch.setattr(config, "_load_settings_from_file", mock_load) config._init_settings() - assert config.LOCAL_DATA_PATH == str(data_root) + assert "local_data_path" not in config.SETTING + assert config.LOCAL_ROOT == str(data_root) # Restore monkeypatch.setattr(config, "_load_settings_from_file", original_load) config._init_settings() - def test_old_format_still_works(self, monkeypatch, tmp_path): - """Old local_data_path format still initializes correctly.""" + def test_s3_config_from_storage_s3(self, monkeypatch, tmp_path): + """MINIO_PARAM and FS are initialized from storage.s3, not minio.""" from hydrodatasource.configs import config - data_root = str(tmp_path / "data") + data_root = tmp_path / "data" def mock_load(): return { - "local_data_path": { - "root": data_root, - "datasets-origin": os.path.join(data_root, "datasets-origin"), - "datasets-interim": os.path.join(data_root, "datasets-interim"), - "cache": os.path.join(data_root, ".cache"), + "storage": { + "local": {"root": str(data_root)}, + "s3": { + "endpoint_url": "http://s3.example.com:9000", + "key": "testkey", + "secret": "testsecret", + }, } } @@ -280,7 +248,29 @@ def mock_load(): monkeypatch.setattr(config, "_load_settings_from_file", mock_load) config._init_settings() - assert config.LOCAL_DATA_PATH == data_root + assert config.MINIO_PARAM["endpoint_url"] == "http://s3.example.com:9000" + assert config.MINIO_PARAM["key"] == "testkey" + assert config.MINIO_PARAM["secret"] == "testsecret" + + # Restore + monkeypatch.setattr(config, "_load_settings_from_file", original_load) + config._init_settings() + + def test_s3_config_empty_when_no_s3_section(self, monkeypatch, tmp_path): + """MINIO_PARAM is empty dict, FS is None when no storage.s3.""" + from hydrodatasource.configs import config + + data_root = tmp_path / "data" + + def mock_load(): + return {"storage": {"local": {"root": str(data_root)}}} + + original_load = config._load_settings_from_file + monkeypatch.setattr(config, "_load_settings_from_file", mock_load) + config._init_settings() + + assert config.MINIO_PARAM == {} + assert config.FS is None # Restore monkeypatch.setattr(config, "_load_settings_from_file", original_load) @@ -309,3 +299,28 @@ def mock_load(): # Restore monkeypatch.setattr(config, "_load_settings_from_file", original_load) config._init_settings() + + def test_empty_config_creates_storage_format(self, monkeypatch): + """When no config file exists, SETTING uses storage.* format, not local_data_path.""" + from hydrodatasource.configs import config + + def mock_load(): + return {} + + original_load = config._load_settings_from_file + monkeypatch.setattr(config, "_load_settings_from_file", mock_load) + config._init_settings() + + assert "local_data_path" not in config.SETTING, ( + "SETTING must not contain local_data_path after bridge removal" + ) + assert "storage" in config.SETTING, ( + "SETTING must have storage key even when config file is empty" + ) + assert config.SETTING["storage"]["local"]["root"], ( + "storage.local.root must have a fallback value" + ) + + # Restore + monkeypatch.setattr(config, "_load_settings_from_file", original_load) + config._init_settings() diff --git a/tests/test_data_source.py b/tests/test_data_source.py index 72bae88..ab12a03 100644 --- a/tests/test_data_source.py +++ b/tests/test_data_source.py @@ -16,57 +16,41 @@ import pandas as pd import pytest import xarray as xr -from hydrodatasource.configs.config import CACHE_DIR, SETTING +from hydrodatasource.configs.config import CACHE_DIR from hydrodatasource.reader.data_source import ( SelfMadeForecastDataset, SelfMadeHydroDataset, ) +_SELFMADE_DATA = os.environ.get("HDS_SELFMADE_DATA") + @pytest.fixture def one_hour_dataset(): - # local - selfmadehydrodataset_path = SETTING["local_data_path"]["datasets-interim"] - # minio - # selfmadehydrodataset_path = "s3://basins-interim" - return SelfMadeHydroDataset( - data_path=selfmadehydrodataset_path, dataset_name="FDSources", time_unit=["1h"] - ) + if not _SELFMADE_DATA: + pytest.skip("Set HDS_SELFMADE_DATA env var") + return SelfMadeHydroDataset(uri=_SELFMADE_DATA, time_unit=["1h"]) @pytest.fixture def three_hour_dataset(): - # local - selfmadehydrodataset_path = SETTING["local_data_path"]["datasets-interim"] - # minio - # selfmadehydrodataset_path = "s3://basins-interim" - return SelfMadeHydroDataset( - data_path=selfmadehydrodataset_path, dataset_name="FDSources", time_unit=["3h"] - ) + if not _SELFMADE_DATA: + pytest.skip("Set HDS_SELFMADE_DATA env var") + return SelfMadeHydroDataset(uri=_SELFMADE_DATA, time_unit=["3h"]) @pytest.fixture def one_day_dataset(): - # local - selfmadehydrodataset_path = SETTING["local_data_path"]["datasets-interim"] - # minio - # selfmadehydrodataset_path = "s3://basins-interim" - return SelfMadeHydroDataset( - data_path=selfmadehydrodataset_path, dataset_name="FDSources" - ) + if not _SELFMADE_DATA: + pytest.skip("Set HDS_SELFMADE_DATA env var") + return SelfMadeHydroDataset(uri=_SELFMADE_DATA) @pytest.fixture def eight_day_dataset(): - # local - selfmadehydrodataset_path = SETTING["local_data_path"]["datasets-interim"] - # minio - # selfmadehydrodataset_path = "s3://basins-interim" - return SelfMadeHydroDataset( - data_path=selfmadehydrodataset_path, - dataset_name="FDSources", - time_unit=["8D"], - ) + if not _SELFMADE_DATA: + pytest.skip("Set HDS_SELFMADE_DATA env var") + return SelfMadeHydroDataset(uri=_SELFMADE_DATA, time_unit=["8D"]) def test_selfmadehydrodataset_get_name(one_day_dataset): @@ -322,13 +306,7 @@ def one_day_forecast_dataset(tmpdir, mocker): attr_file = attr_dir.join("attributes.csv") attr_file.write("basin_id,area\nbasin_1,100\nbasin_2,200\n") - # local - selfmadehydrodataset_path = str(tmpdir) - # minio - # selfmadehydrodataset_path = "s3://basins-interim" - return SelfMadeForecastDataset( - data_path=selfmadehydrodataset_path, dataset_name="FDSources" - ) + return SelfMadeForecastDataset(uri=str(fdsources_dir)) def test_read_forecast_multiple_basins_all_exist(mocker, one_day_forecast_dataset): @@ -358,23 +336,19 @@ def test_read_forecast_multiple_basins_all_exist(mocker, one_day_forecast_datase def test_start_hour_in_a_day_validation(): """Test that start_hour_in_a_day validation works correctly for time interval format.""" - selfmadehydrodataset_path = SETTING["local_data_path"]["datasets-interim"] + data_path = os.environ.get("HDS_SELFMADE_DATA") + if not data_path: + pytest.skip("Set HDS_SELFMADE_DATA env var") # Test: Non-standard time_unit like 6h should be rejected at init level # (SelfMadeHydroDataset only supports '1h', '3h', '1D', '8D') with pytest.raises(ValueError) as excinfo: - dataset_6h = SelfMadeHydroDataset( - data_path=selfmadehydrodataset_path, - dataset_name="FDSources", - time_unit=["6h"], - ) + dataset_6h = SelfMadeHydroDataset(uri=data_path, time_unit=["6h"]) assert "time_unit must be one of" in str(excinfo.value) # Test that the cache_timeseries_xrdataset validation catches unsupported intervals # when they are passed directly to the function - dataset = SelfMadeHydroDataset( - data_path=selfmadehydrodataset_path, dataset_name="FDSources", time_unit=["3h"] - ) + dataset = SelfMadeHydroDataset(uri=data_path, time_unit=["3h"]) dataset.trange4cache = None dataset.offset_to_utc = False @@ -403,11 +377,11 @@ def test_start_hour_in_a_day_validation(): def test_start_hour_in_a_day_time_range(): """Test that start_hour_in_a_day correctly sets the time range.""" - selfmadehydrodataset_path = SETTING["local_data_path"]["datasets-interim"] + data_path = os.environ.get("HDS_SELFMADE_DATA") + if not data_path: + pytest.skip("Set HDS_SELFMADE_DATA env var") - dataset = SelfMadeHydroDataset( - data_path=selfmadehydrodataset_path, dataset_name="FDSources", time_unit=["3h"] - ) + dataset = SelfMadeHydroDataset(uri=data_path, time_unit=["3h"]) dataset.trange4cache = None dataset.offset_to_utc = False @@ -429,11 +403,11 @@ def test_start_hour_in_a_day_time_range(): def test_start_hour_in_a_day_data_alignment(mocker): """Test that data alignment validation works correctly.""" - selfmadehydrodataset_path = SETTING["local_data_path"]["datasets-interim"] + data_path = os.environ.get("HDS_SELFMADE_DATA") + if not data_path: + pytest.skip("Set HDS_SELFMADE_DATA env var") - dataset = SelfMadeHydroDataset( - data_path=selfmadehydrodataset_path, dataset_name="FDSources", time_unit=["3h"] - ) + dataset = SelfMadeHydroDataset(uri=data_path, time_unit=["3h"]) dataset.offset_to_utc = False # Mock data with hours starting at 00:00 (0, 3, 6, 9, 12, 15, 18, 21) diff --git a/tests/test_dmca_esr.py b/tests/test_dmca_esr.py index 85a16ce..1dbc411 100644 --- a/tests/test_dmca_esr.py +++ b/tests/test_dmca_esr.py @@ -15,15 +15,21 @@ import numpy as np from pint import UnitRegistry -from hydrodataset import CamelsUs +try: + from hydrodataset import CamelsUs +except ImportError: + pytest.skip("CamelsUs not available in installed hydrodataset", allow_module_level=True) -from hydrodatasource.configs.config import SETTING +from hydrodatasource.configs.config import get_local_root from hydrodatasource.processor.dmca_esr import * from hydroutils import hydro_units def test_rainfall_runoff_event_identify(): - camels = CamelsUs(os.path.join(SETTING["local_data_path"]["datasets-origin"])) + root = get_local_root() + if root is None: + pytest.skip("local root not configured") + camels = CamelsUs(str(root / "datasets-origin")) gage_ids = camels.read_object_ids() ureg = UnitRegistry() diff --git a/tests/test_gages.py b/tests/test_gages.py index 9a6fbee..e638a82 100644 --- a/tests/test_gages.py +++ b/tests/test_gages.py @@ -12,18 +12,18 @@ import pytest import pandas as pd from hydrodatasource.reader.gages import Gages -from hydrodatasource.configs.config import CACHE_DIR, SETTING +from hydrodatasource.configs.config import CACHE_DIR pytestmark = pytest.mark.internal_data +_GAGES_DATA = os.environ.get("HDS_GAGES_DATA") + @pytest.fixture def gages_dataset(): - # local - gages_path = os.path.join(SETTING["local_data_path"]["datasets-interim"]) - # minio - # gages_path = "s3://basins-interim" - return Gages(data_path=gages_path, dataset_name="hydrodl-reservoir-jh-paper") + if not _GAGES_DATA: + pytest.skip("Set HDS_GAGES_DATA env var") + return Gages(_GAGES_DATA) def test_gages_read_site_info(gages_dataset): diff --git a/tests/test_reader_uri_constructors.py b/tests/test_reader_uri_constructors.py index 84bbc44..22ce8f7 100644 --- a/tests/test_reader_uri_constructors.py +++ b/tests/test_reader_uri_constructors.py @@ -19,74 +19,70 @@ class TestHydroDataBase: - """Test the HydroData abstract base class constructor.""" + """Test the HydroData abstract base class constructor. + + Only uri= (positional) is accepted. No data_path, dataset_name, or data_folder. + """ def test_uri_construction_sets_data_source_dir(self): - """uri kwarg sets data_source_dir directly.""" + """uri sets data_source_dir directly.""" from hydrodatasource.reader.data_source import HydroData - ds = HydroData(uri="/tmp/test/dataset") + ds = HydroData("/tmp/test/dataset") assert ds.data_source_dir == "/tmp/test/dataset" def test_uri_construction_sets_dataset_name_from_uri(self): - """dataset_name defaults to the last component of the URI path.""" + """dataset_name is always derived from the URI basename.""" from hydrodatasource.reader.data_source import HydroData - ds = HydroData(uri="/tmp/test/my_dataset") + ds = HydroData("/tmp/test/my_dataset") assert ds.dataset_name == "my_dataset" - def test_uri_with_explicit_dataset_name(self): - """Explicit dataset_name is preserved when uri is provided.""" - from hydrodatasource.reader.data_source import HydroData - - ds = HydroData(uri="/tmp/test/data", dataset_name="custom_label") - assert ds.data_source_dir == "/tmp/test/data" - assert ds.dataset_name == "custom_label" - def test_uri_as_pathlib_path(self): """uri accepts pathlib.Path objects (via str conversion).""" from hydrodatasource.reader.data_source import HydroData - ds = HydroData(uri=Path("/tmp/test/dataset")) + ds = HydroData(Path("/tmp/test/dataset")) assert ds.data_source_dir.endswith("dataset") def test_uri_with_s3_prefix(self): """uri works with s3:// URIs.""" from hydrodatasource.reader.data_source import HydroData - ds = HydroData(uri="s3://bucket/prefix/dataset") + ds = HydroData("s3://bucket/prefix/dataset") assert ds.data_source_dir == "s3://bucket/prefix/dataset" assert ds.dataset_name == "dataset" - def test_legacy_data_path_and_dataset_name(self): - """Legacy construction still works with data_path + dataset_name.""" + def test_uri_is_required(self): + """HydroData() without uri raises TypeError.""" from hydrodatasource.reader.data_source import HydroData - ds = HydroData(data_path="/parent", dataset_name="child") - assert ds.data_source_dir == os.path.join("/parent", "child") - assert ds.dataset_name == "child" + with pytest.raises(TypeError): + HydroData() - def test_uri_takes_precedence_over_data_path(self): - """When uri is provided, data_path and dataset_name are ignored.""" + def test_data_path_rejected(self): + """data_path= is no longer accepted — raises TypeError.""" from hydrodatasource.reader.data_source import HydroData - ds = HydroData( - uri="/explicit/uri/path", - data_path="/ignored/parent", - dataset_name="ignored_name", - ) - assert ds.data_source_dir == "/explicit/uri/path" - assert ds.dataset_name == "ignored_name" + with pytest.raises(TypeError): + HydroData(data_path="/parent") + + def test_dataset_name_rejected(self): + """dataset_name= is no longer accepted as a parameter.""" + from hydrodatasource.reader.data_source import HydroData + + with pytest.raises(TypeError): + HydroData("/path", dataset_name="custom") # ── Pattern B classes (bypass super().__init__, set data_source_dir directly) ─ class TestPatternBConstructors: - """Test classes that bypass super().__init__(). + """Test classes that call super().__init__() following the unified pattern. - Grdc, RainfallReader, Crd, RsvrInflowReader — these set - data_source_dir directly and DON'T call super().__init__(). + Grdc, RainfallReader, Crd, RsvrInflowReader — all accept uri as positional + and delegate to HydroData.__init__(). """ @pytest.fixture @@ -124,59 +120,31 @@ def mock_read_site_info(self): yield def test_grdc_uri_construction(self, mock_read_site_info): - """Grdc accepts uri kwarg and sets data_source_dir.""" + """Grdc accepts uri as positional argument.""" from hydrodatasource.reader.grdc import Grdc - ds = Grdc(uri="/data/grdc") - assert ds.data_source_dir == "/data/grdc" - - def test_grdc_legacy_construction(self, mock_read_site_info): - """Grdc still works with legacy data_path arg.""" - from hydrodatasource.reader.grdc import Grdc - - ds = Grdc(data_path="/data/grdc") + ds = Grdc("/data/grdc") assert ds.data_source_dir == "/data/grdc" def test_rainfall_reader_uri_construction(self, mock_read_site_info): - """RainfallReader accepts uri kwarg.""" + """RainfallReader accepts uri as positional argument.""" from hydrodatasource.reader.rainfall_reader import RainfallReader - ds = RainfallReader(uri="/data/rainfall") - assert ds.data_source_dir == "/data/rainfall" - - def test_rainfall_reader_legacy_construction(self, mock_read_site_info): - """RainfallReader still works with legacy data_folder arg.""" - from hydrodatasource.reader.rainfall_reader import RainfallReader - - ds = RainfallReader(data_folder="/data/rainfall") + ds = RainfallReader("/data/rainfall") assert ds.data_source_dir == "/data/rainfall" def test_crd_uri_construction(self, mock_read_site_info): - """Crd accepts uri kwarg.""" - from hydrodatasource.reader.reservoir_datasets import Crd - - ds = Crd(uri="/data/crd") - assert ds.data_source_dir == "/data/crd" - - def test_crd_legacy_construction(self, mock_read_site_info): - """Crd still works with legacy data_path arg.""" + """Crd accepts uri as positional argument.""" from hydrodatasource.reader.reservoir_datasets import Crd - ds = Crd(data_path="/data/crd") + ds = Crd("/data/crd") assert ds.data_source_dir == "/data/crd" def test_rsvr_inflow_uri_construction(self, mock_read_site_info): - """RsvrInflowReader accepts uri kwarg.""" + """RsvrInflowReader accepts uri as positional argument.""" from hydrodatasource.reader.rsvr_inflow_reader import RsvrInflowReader - ds = RsvrInflowReader(uri="/data/rsvr_inflow") - assert ds.data_source_dir == "/data/rsvr_inflow" - - def test_rsvr_inflow_legacy_construction(self, mock_read_site_info): - """RsvrInflowReader still works with legacy data_folder arg.""" - from hydrodatasource.reader.rsvr_inflow_reader import RsvrInflowReader - - ds = RsvrInflowReader(data_folder="/data/rsvr_inflow") + ds = RsvrInflowReader("/data/rsvr_inflow") assert ds.data_source_dir == "/data/rsvr_inflow" @pytest.mark.parametrize( @@ -195,19 +163,19 @@ def test_pattern_b_readers_have_dataset_name( if reader_cls_factory == "grdc": from hydrodatasource.reader.grdc import Grdc - ds = Grdc(uri=f"/data/{expected_name}") + ds = Grdc(f"/data/{expected_name}") elif reader_cls_factory == "rainfall": from hydrodatasource.reader.rainfall_reader import RainfallReader - ds = RainfallReader(uri=f"/data/{expected_name}") + ds = RainfallReader(f"/data/{expected_name}") elif reader_cls_factory == "crd": from hydrodatasource.reader.reservoir_datasets import Crd - ds = Crd(uri=f"/data/{expected_name}") + ds = Crd(f"/data/{expected_name}") elif reader_cls_factory == "rsvrinflow": from hydrodatasource.reader.rsvr_inflow_reader import RsvrInflowReader - ds = RsvrInflowReader(uri=f"/data/{expected_name}") + ds = RsvrInflowReader(f"/data/{expected_name}") assert hasattr(ds, "dataset_name"), f"{reader_cls_factory}: no dataset_name attr" assert ds.dataset_name == expected_name, ( @@ -416,17 +384,16 @@ def test_time_unit_validation(self): time_unit=["invalid_unit"], ) - def test_legacy_construction_with_minimal_data(self, minimal_dataset): - """Legacy data_path + dataset_name still works.""" + def test_uri_construction_from_parent_dir(self, minimal_dataset): + """uri pointing directly to the dataset directory.""" from hydrodatasource.reader.data_source import SelfMadeHydroDataset ds = SelfMadeHydroDataset( - data_path=str(minimal_dataset.parent), - dataset_name="minimal_dataset", + uri=str(minimal_dataset), time_unit=["1D"], ) assert ds.dataset_name == "minimal_dataset" - assert str(minimal_dataset) in ds.data_source_dir + assert ds.data_source_dir == str(minimal_dataset) class TestS3UriPathCheck: diff --git a/tests/test_tg_hydro_datasource.py b/tests/test_tg_hydro_datasource.py index a7af500..8dbf50d 100644 --- a/tests/test_tg_hydro_datasource.py +++ b/tests/test_tg_hydro_datasource.py @@ -165,7 +165,7 @@ def tg_dataset(tmp_path): _create_unit_info(base) _create_attributes(base) - ds = TgHydroDatasource(data_path=str(tmp_path), dataset_name=dataset_name, time_unit=["1D"]) + ds = TgHydroDatasource(uri=str(base), time_unit=["1D"]) return ds diff --git a/uv.lock b/uv.lock index 31a31ab..bb9a48f 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.14'", @@ -2481,6 +2481,7 @@ dev = [ { name = "flake8" }, { name = "pytest" }, { name = "pytest-asyncio" }, + { name = "pytest-dotenv" }, { name = "pytest-mock" }, { name = "pytest-runner" }, { name = "tox" }, @@ -2509,6 +2510,11 @@ docs = [ { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] +[package.dev-dependencies] +dev = [ + { name = "pytest-dotenv" }, +] + [package.metadata] requires-dist = [ { name = "aqua-fetch", extras = ["all"] }, @@ -2551,6 +2557,7 @@ requires-dist = [ { name = "pymdown-extensions", marker = "extra == 'docs'" }, { name = "pytest", marker = "extra == 'dev'" }, { name = "pytest-asyncio", marker = "extra == 'dev'" }, + { name = "pytest-dotenv", marker = "extra == 'dev'" }, { name = "pytest-mock", marker = "extra == 'dev'" }, { name = "pytest-runner", marker = "extra == 'dev'" }, { name = "pywavelets" }, @@ -2568,6 +2575,9 @@ requires-dist = [ ] provides-extras = ["dev", "docs"] +[package.metadata.requires-dev] +dev = [{ name = "pytest-dotenv", specifier = ">=0.5.2" }] + [[package]] name = "hydroerr" version = "1.24" @@ -6015,6 +6025,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/93/2fa34714b7a4ae72f2f8dad66ba17dd9a2c793220719e736dda28b7aec27/pytest_asyncio-1.2.0-py3-none-any.whl", hash = "sha256:8e17ae5e46d8e7efe51ab6494dd2010f4ca8dae51652aa3c8d55acf50bfb2e99", size = 15095, upload-time = "2025-09-12T07:33:52.639Z" }, ] +[[package]] +name = "pytest-dotenv" +version = "0.5.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, + { name = "python-dotenv" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cd/b0/cafee9c627c1bae228eb07c9977f679b3a7cb111b488307ab9594ba9e4da/pytest-dotenv-0.5.2.tar.gz", hash = "sha256:2dc6c3ac6d8764c71c6d2804e902d0ff810fa19692e95fe138aefc9b1aa73732", size = 3782, upload-time = "2020-06-16T12:38:03.4Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/da/9da67c67b3d0963160e3d2cbc7c38b6fae342670cc8e6d5936644b2cf944/pytest_dotenv-0.5.2-py3-none-any.whl", hash = "sha256:40a2cece120a213898afaa5407673f6bd924b1fa7eafce6bda0e8abffe2f710f", size = 3993, upload-time = "2020-06-16T12:38:01.139Z" }, +] + [[package]] name = "pytest-mock" version = "3.15.1" @@ -6048,6 +6071,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, ] +[[package]] +name = "python-dotenv" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, +] + [[package]] name = "pytkdocs" version = "0.16.5" From 43de2b0f8ecdae9e6d358b8b5706dc1daf4e62eb Mon Sep 17 00:00:00 2001 From: owen Date: Tue, 23 Jun 2026 22:16:37 +0800 Subject: [PATCH 22/34] refactor: remove library-level datasets.yml, clarify YAML is user-layer config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Datasets registry lives in _HDS_DATASETS (in-code, Layer 2 of 3-layer cascade). configs/datasets.yml is a Layer 3 user override — the library does not ship one. Changes: - Delete configs/datasets.yml (redundant with _HDS_DATASETS) - Update data_resolver.py comments to document the 3-layer architecture - Add tests: resolution without YAML, _HDS_DATASETS entry verification All 69 relevant tests pass. --- configs/datasets.yml | 12 -------- hydrodatasource/configs/data_resolver.py | 10 ++++++- tests/test_data_resolver.py | 37 ++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 13 deletions(-) delete mode 100644 configs/datasets.yml diff --git a/configs/datasets.yml b/configs/datasets.yml deleted file mode 100644 index 5d658e5..0000000 --- a/configs/datasets.yml +++ /dev/null @@ -1,12 +0,0 @@ -# Hydrodatasource dataset registry. -# Maps dataset id -> reader alias + relative path. -# Format: datasets..reader (str), datasets..path (relative str) -# -# Rules (from hydromodel ADR 0001): -# - path must be relative (no absolute, no URI, no .. traversal) -# - reader must name a registered alias in data_resolver.READER_ALIASES - -datasets: - songliao_event: - reader: floodevent - path: projects/songliao/event diff --git a/hydrodatasource/configs/data_resolver.py b/hydrodatasource/configs/data_resolver.py index b5b4845..fa80f3e 100644 --- a/hydrodatasource/configs/data_resolver.py +++ b/hydrodatasource/configs/data_resolver.py @@ -97,7 +97,15 @@ **_HDS_READER_ALIASES, } -# ── In-memory dataset registry (no YAML file needed) ───────────────────── +# ── Dataset registry (in-code, not YAML) ────────────────────────────────── +# +# The library's dataset registry lives here in code. This is the authoritative +# source for datasets served by this package (Layer 2 in the 3-layer cascade). +# +# YAML files (configs/datasets.yml) are a Layer 3 **user override** — external +# projects using hydrodatasource or hydrodataset can provide their own +# configs/datasets.yml to customize or extend the registry. The library itself +# does not ship a default YAML. _HDS_DATASETS: Dict[str, Dict[str, str]] = { "songliao_event": { diff --git a/tests/test_data_resolver.py b/tests/test_data_resolver.py index fa553f4..e6561b9 100644 --- a/tests/test_data_resolver.py +++ b/tests/test_data_resolver.py @@ -181,3 +181,40 @@ def test_resolve_with_local_root_override(self, tmp_path): ) expected = str(dataset_dir) assert uri == expected + + def test_resolves_via_injected_registry_without_yaml(self, tmp_path, monkeypatch): + """songliao_event resolves via _HDS_DATASETS injection (no YAML needed). + + The in-code _HDS_DATASETS registry is the library's source of truth. + YAML files (configs/datasets.yml) are for user projects — the library + does not ship one. Resolution must work without any YAML file present. + """ + from hydrodatasource.configs.data_resolver import resolve_data_path + + # Create temp data directory (no YAML anywhere) + data_dir = tmp_path / "data" + data_dir.mkdir(parents=True) + dataset_dir = data_dir / "projects" / "songliao" / "event" + dataset_dir.mkdir(parents=True) + + monkeypatch.setattr( + "hydrodataset.configs.data_resolver.get_local_root", + lambda: data_dir, + ) + + # Pass a project_root that has NO configs/datasets.yml + # Resolution must succeed via _HDS_DATASETS injection alone + uri = resolve_data_path( + "songliao_event", source="local", project_root=str(tmp_path) + ) + expected = str(dataset_dir) + assert uri == expected + + def test_extra_registry_contains_songliao_event(self): + """_HDS_DATASETS hardcoded registry contains the songliao_event entry.""" + from hydrodatasource.configs.data_resolver import _HDS_DATASETS + + assert "songliao_event" in _HDS_DATASETS + spec = _HDS_DATASETS["songliao_event"] + assert spec["reader"] == "floodevent" + assert spec["path"] == "projects/songliao/event" From d8ee24b91bc85dc2976ac7c7a136bc27edca819e Mon Sep 17 00:00:00 2001 From: owen Date: Wed, 24 Jun 2026 16:53:05 +0800 Subject: [PATCH 23/34] refactor: rename _HDS_DATASETS to HDS_DATASETS, add __all__ Make the dataset registry dict a public name since downstream code may need to inspect or extend it. Add __all__ to explicitly control what's exported from the data_resolver module. --- hydrodatasource/configs/data_resolver.py | 11 +++++++++-- tests/test_data_resolver.py | 14 +++++++------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/hydrodatasource/configs/data_resolver.py b/hydrodatasource/configs/data_resolver.py index fa80f3e..11e3a36 100644 --- a/hydrodatasource/configs/data_resolver.py +++ b/hydrodatasource/configs/data_resolver.py @@ -31,6 +31,13 @@ resolve_data_path as _hd_resolve_data_path, ) +__all__ = [ + "DatasetResolutionError", + "HDS_DATASETS", + "READER_ALIASES", + "resolve_data_path", +] + # ── hydrodatasource-specific reader aliases ────────────────────────────── _HDS_READER_ALIASES: Dict[str, Dict[str, str]] = { @@ -107,7 +114,7 @@ # configs/datasets.yml to customize or extend the registry. The library itself # does not ship a default YAML. -_HDS_DATASETS: Dict[str, Dict[str, str]] = { +HDS_DATASETS: Dict[str, Dict[str, str]] = { "songliao_event": { "reader": "floodevent", "path": "projects/songliao/event", @@ -155,7 +162,7 @@ def resolve_data_path( source=source, project_root=project_root, local_root=Path(local_root) if local_root else None, - extra_registry_dicts=[_HDS_DATASETS], + extra_registry_dicts=[HDS_DATASETS], extra_reader_aliases=_HDS_READER_ALIASES, ) return str(result) diff --git a/tests/test_data_resolver.py b/tests/test_data_resolver.py index e6561b9..015f842 100644 --- a/tests/test_data_resolver.py +++ b/tests/test_data_resolver.py @@ -183,9 +183,9 @@ def test_resolve_with_local_root_override(self, tmp_path): assert uri == expected def test_resolves_via_injected_registry_without_yaml(self, tmp_path, monkeypatch): - """songliao_event resolves via _HDS_DATASETS injection (no YAML needed). + """songliao_event resolves via HDS_DATASETS injection (no YAML needed). - The in-code _HDS_DATASETS registry is the library's source of truth. + The in-code HDS_DATASETS registry is the library's source of truth. YAML files (configs/datasets.yml) are for user projects — the library does not ship one. Resolution must work without any YAML file present. """ @@ -203,7 +203,7 @@ def test_resolves_via_injected_registry_without_yaml(self, tmp_path, monkeypatch ) # Pass a project_root that has NO configs/datasets.yml - # Resolution must succeed via _HDS_DATASETS injection alone + # Resolution must succeed via HDS_DATASETS injection alone uri = resolve_data_path( "songliao_event", source="local", project_root=str(tmp_path) ) @@ -211,10 +211,10 @@ def test_resolves_via_injected_registry_without_yaml(self, tmp_path, monkeypatch assert uri == expected def test_extra_registry_contains_songliao_event(self): - """_HDS_DATASETS hardcoded registry contains the songliao_event entry.""" - from hydrodatasource.configs.data_resolver import _HDS_DATASETS + """HDS_DATASETS hardcoded registry contains the songliao_event entry.""" + from hydrodatasource.configs.data_resolver import HDS_DATASETS - assert "songliao_event" in _HDS_DATASETS - spec = _HDS_DATASETS["songliao_event"] + assert "songliao_event" in HDS_DATASETS + spec = HDS_DATASETS["songliao_event"] assert spec["reader"] == "floodevent" assert spec["path"] == "projects/songliao/event" From 956776dac3903616793f3d19cd00923c8f658f1a Mon Sep 17 00:00:00 2001 From: owen Date: Thu, 25 Jun 2026 22:19:37 +0800 Subject: [PATCH 24/34] feat: add open_dataset wrapper and align data_resolver with ResolverContext API Expose a thin open_dataset factory that injects HDS registry/aliases so callers can instantiate readers in one line, and migrate resolve_data_path to the shared ctx-based contract. Co-authored-by: Cursor --- .gitignore | 3 + hydrodatasource/__init__.py | 2 + hydrodatasource/configs/data_resolver.py | 139 ++++++++++--- tests/test_data_resolver.py | 246 +++++++++++++++++++---- tests/test_package_exports.py | 13 +- 5 files changed, 332 insertions(+), 71 deletions(-) diff --git a/.gitignore b/.gitignore index a873603..11e12db 100644 --- a/.gitignore +++ b/.gitignore @@ -171,3 +171,6 @@ CLAUDE.md .claude/ hydrodatasource/_version.py GEMINI.md + +# Cursor code graph +.codegraph/ diff --git a/hydrodatasource/__init__.py b/hydrodatasource/__init__.py index 012e30a..758d1f4 100644 --- a/hydrodatasource/__init__.py +++ b/hydrodatasource/__init__.py @@ -11,6 +11,8 @@ # Re-export config APIs for convenience from .configs.data_resolver import ( READER_ALIASES, + ResolverContext, resolve_data_path, + open_dataset, DatasetResolutionError, ) diff --git a/hydrodatasource/configs/data_resolver.py b/hydrodatasource/configs/data_resolver.py index 11e3a36..736c0b9 100644 --- a/hydrodatasource/configs/data_resolver.py +++ b/hydrodatasource/configs/data_resolver.py @@ -2,32 +2,35 @@ Reader alias registry and data path resolution for hydrodatasource. Defines READER_ALIASES for all hydrodatasource reader classes and provides -resolve_data_path() following the same contract as hydromodel's ADR 0001. +resolve_data_path() and open_dataset() following the same contract as +hydromodel's ADR 0001. This module is a thin wrapper around hydrodataset.configs.data_resolver. -It only uses the PUBLIC API of hydrodataset (resolve_data_path, READER_ALIASES, -DatasetResolutionError). The hydrodatasource-specific parts are injected via -extra_registry_dicts and extra_reader_aliases parameters. +It only uses the PUBLIC API of hydrodataset (resolve_data_path, open_dataset, +READER_ALIASES, DatasetResolutionError). The hydrodatasource-specific parts are +injected via extra_registry_dicts and extra_reader_aliases parameters. Usage: from hydrodatasource.configs.data_resolver import ( READER_ALIASES, resolve_data_path, + open_dataset, DatasetResolutionError, ) - uri = resolve_data_path("songliao_event", local_root="/data/hydro") - ds = FloodEventDatasource(uri=uri) + uri = resolve_data_path("songliao_event") + ds = open_dataset("songliao_event") """ from __future__ import annotations -from pathlib import Path -from typing import Dict, Optional +from typing import Any, Dict, Literal, Optional from hydrodataset.configs.data_resolver import ( DatasetResolutionError, READER_ALIASES as _HD_READER_ALIASES, + ResolverContext, + open_dataset as _hd_open_dataset, resolve_data_path as _hd_resolve_data_path, ) @@ -35,6 +38,8 @@ "DatasetResolutionError", "HDS_DATASETS", "READER_ALIASES", + "ResolverContext", + "open_dataset", "resolve_data_path", ] @@ -122,17 +127,51 @@ } +def _with_hds_extras(ctx: Optional[ResolverContext]) -> ResolverContext: + """Return a ResolverContext with HDS datasets and reader aliases injected. + + When *ctx* is None a fresh context containing only the HDS extras is + returned. When *ctx* is provided, the HDS datasets and aliases are merged + on top of any extras already present in the caller's context (HDS values + take precedence so that hydrodatasource readers are always resolvable). + + Args: + ctx: Caller-supplied resolver context, or None. + + Returns: + A new ResolverContext with HDS_DATASETS and _HDS_READER_ALIASES merged + in. + """ + if ctx is None: + return ResolverContext( + extra_registry_dicts=[HDS_DATASETS], + extra_reader_aliases=_HDS_READER_ALIASES, + ) + # Merge caller extras (first) with HDS defaults (HDS wins on overlap). + merged_registry_dicts = list(ctx.extra_registry_dicts or []) + merged_registry_dicts.append(HDS_DATASETS) + merged_reader_aliases = dict(ctx.extra_reader_aliases or {}) + merged_reader_aliases.update(_HDS_READER_ALIASES) + return ResolverContext( + project_root=ctx.project_root, + storage=ctx.storage, + registry=ctx.registry, + extra_registry_dicts=merged_registry_dicts, + extra_reader_aliases=merged_reader_aliases, + ) + + def resolve_data_path( dataset_id: str, *, - source: str = "local", - project_root: Optional[str] = None, - local_root: Optional[str] = None, + source: Literal["local", "cloud"] = "local", + ctx: Optional[ResolverContext] = None, ) -> str: """Resolve a dataset id to an absolute data path (URI). Thin wrapper around hydrodataset's resolve_data_path that injects - hydrodatasource-specific datasets and reader aliases. + hydrodatasource-specific datasets (HDS_DATASETS) and reader aliases + (_HDS_READER_ALIASES) into the resolution context. Parameters ---------- @@ -140,12 +179,11 @@ def resolve_data_path( Dataset identifier from the registry (e.g. 'camels_us', 'songliao_event'). source : str Storage backend: 'local' or 'cloud'. - project_root : str, optional - Root of the calling project (for finding configs/datasets.yml). - Defaults to current working directory. - local_root : str, optional - Override storage.local.root for project-level root override. - When provided, skips reading storage.local.root from settings. + ctx : ResolverContext, optional + Resolution context bundling project_root, storage config, registry + overrides, and extra aliases. When None (default), a new context + is created with HDS_DATASETS and _HDS_READER_ALIASES injected. + When provided, caller's extras are merged with HDS defaults. Returns ------- @@ -156,13 +194,62 @@ def resolve_data_path( ------ DatasetResolutionError If any resolution step fails. + + Examples + -------- + >>> # Default: zero-boilerplate, reads ~/hydro_setting.yml + >>> uri = resolve_data_path("songliao_event") + + >>> # Custom storage root via ResolverContext + >>> ctx = ResolverContext(storage={"local": {"root": "/custom/data"}}) + >>> uri = resolve_data_path("songliao_event", ctx=ctx) + """ + return _hd_resolve_data_path(dataset_id, source=source, ctx=_with_hds_extras(ctx)) + + +def open_dataset( + dataset_id: str, + *, + source: Literal["local", "cloud"] = "local", + ctx: Optional[ResolverContext] = None, + **reader_kwargs: Any, +): + """Resolve a dataset id and return an instantiated reader object. + + Thin wrapper around hydrodataset's open_dataset that injects + hydrodatasource-specific datasets (HDS_DATASETS) and reader aliases + (_HDS_READER_ALIASES) into the resolution context. Both hydrodataset + datasets (e.g. ``'camels_us'``) and hydrodatasource datasets + (e.g. ``'songliao_event'``) are supported via a single call. + + Parameters + ---------- + dataset_id : str + Dataset identifier from the registry (e.g. 'camels_us', 'songliao_event'). + source : str + Storage backend: 'local' or 'cloud'. + ctx : ResolverContext, optional + Resolution context. When None, HDS defaults are used automatically. + **reader_kwargs + Extra keyword arguments forwarded to the reader constructor + (e.g. ``time_unit=["1D"]`` for ``SelfMadeHydroDataset``). + + Returns + ------- + object + An instance of the reader class registered for *dataset_id*. + + Raises + ------ + DatasetResolutionError + If any resolution step fails. + + Examples + -------- + >>> ds = open_dataset("songliao_event") + >>> ds = open_dataset("camels_us", source="cloud") + >>> ds = open_dataset("selfmade_basin", time_unit=["1D"]) """ - result = _hd_resolve_data_path( - dataset_id, - source=source, - project_root=project_root, - local_root=Path(local_root) if local_root else None, - extra_registry_dicts=[HDS_DATASETS], - extra_reader_aliases=_HDS_READER_ALIASES, + return _hd_open_dataset( + dataset_id, source=source, ctx=_with_hds_extras(ctx), **reader_kwargs ) - return str(result) diff --git a/tests/test_data_resolver.py b/tests/test_data_resolver.py index 015f842..c36fd37 100644 --- a/tests/test_data_resolver.py +++ b/tests/test_data_resolver.py @@ -1,13 +1,15 @@ """Tests for hydrodatasource.configs.data_resolver.""" -import os +import sys import tempfile +import types from pathlib import Path -from unittest.mock import patch import pytest import yaml +from hydrodataset.configs.data_resolver import ResolverContext + class TestReaderAliases: """Verify READER_ALIASES covers all hydrodatasource reader classes.""" @@ -104,67 +106,56 @@ def datasets_yml(self, tmp_path): ) return tmp_path - def test_resolve_local_path(self, temp_root, datasets_yml, monkeypatch): + def test_resolve_local_path(self, temp_root, datasets_yml): """Resolve a dataset to an absolute local path.""" from hydrodatasource.configs.data_resolver import resolve_data_path - monkeypatch.setattr( - "hydrodataset.configs.data_resolver.get_local_root", - lambda: temp_root / "data", + ctx = ResolverContext( + storage={"local": {"root": str(temp_root / "data")}}, + project_root=datasets_yml, ) - uri = resolve_data_path( - "songliao_event", source="local", project_root=str(datasets_yml) - ) + uri = resolve_data_path("songliao_event", source="local", ctx=ctx) expected = str(temp_root / "data" / "projects" / "songliao" / "event") assert uri == expected - def test_resolve_unknown_dataset_raises(self, datasets_yml, monkeypatch, tmp_path): + def test_resolve_unknown_dataset_raises(self, datasets_yml): """Resolving an unknown dataset id should raise.""" from hydrodatasource.configs.data_resolver import ( resolve_data_path, DatasetResolutionError, ) - monkeypatch.setattr( - "hydrodataset.configs.data_resolver.get_local_root", - lambda: tmp_path / "data", - ) + ctx = ResolverContext(project_root=datasets_yml) with pytest.raises(DatasetResolutionError): - resolve_data_path( - "nonexistent_dataset", source="local", project_root=str(datasets_yml) - ) + resolve_data_path("nonexistent_dataset", source="local", ctx=ctx) - def test_invalid_source_raises(self, datasets_yml): + def test_invalid_source_raises(self, datasets_yml, tmp_path): """Invalid source should raise.""" from hydrodatasource.configs.data_resolver import ( resolve_data_path, DatasetResolutionError, ) + ctx = ResolverContext( + storage={"local": {"root": str(tmp_path / "data")}}, + project_root=datasets_yml, + ) + with pytest.raises(DatasetResolutionError): - resolve_data_path( - "songliao_event", source="ftp", project_root=str(datasets_yml) - ) + resolve_data_path("songliao_event", source="ftp", ctx=ctx) - def test_resolve_cloud_s3_uri(self, datasets_yml, monkeypatch): + def test_resolve_cloud_s3_uri(self, datasets_yml): """Resolve a dataset to an S3 URI for cloud source.""" from hydrodatasource.configs.data_resolver import resolve_data_path - monkeypatch.setattr( - "hydrodataset.configs.data_resolver.get_storage_config", - lambda: { - "s3": { - "bucket": "hydro-data", - "prefix": "hydromodel", - } - }, + ctx = ResolverContext( + storage={"s3": {"bucket": "hydro-data", "prefix": "hydromodel"}}, + project_root=datasets_yml, ) - uri = resolve_data_path( - "songliao_event", source="cloud", project_root=str(datasets_yml) - ) + uri = resolve_data_path("songliao_event", source="cloud", ctx=ctx) assert uri == "s3://hydro-data/hydromodel/projects/songliao/event" def test_resolve_with_local_root_override(self, tmp_path): @@ -176,13 +167,15 @@ def test_resolve_with_local_root_override(self, tmp_path): dataset_dir = custom_root / "projects" / "songliao" / "event" dataset_dir.mkdir(parents=True) - uri = resolve_data_path( - "songliao_event", source="local", local_root=str(custom_root) + ctx = ResolverContext( + storage={"local": {"root": str(custom_root)}}, ) + + uri = resolve_data_path("songliao_event", source="local", ctx=ctx) expected = str(dataset_dir) assert uri == expected - def test_resolves_via_injected_registry_without_yaml(self, tmp_path, monkeypatch): + def test_resolves_via_injected_registry_without_yaml(self, tmp_path): """songliao_event resolves via HDS_DATASETS injection (no YAML needed). The in-code HDS_DATASETS registry is the library's source of truth. @@ -197,16 +190,14 @@ def test_resolves_via_injected_registry_without_yaml(self, tmp_path, monkeypatch dataset_dir = data_dir / "projects" / "songliao" / "event" dataset_dir.mkdir(parents=True) - monkeypatch.setattr( - "hydrodataset.configs.data_resolver.get_local_root", - lambda: data_dir, + ctx = ResolverContext( + storage={"local": {"root": str(data_dir)}}, + project_root=tmp_path, ) # Pass a project_root that has NO configs/datasets.yml # Resolution must succeed via HDS_DATASETS injection alone - uri = resolve_data_path( - "songliao_event", source="local", project_root=str(tmp_path) - ) + uri = resolve_data_path("songliao_event", source="local", ctx=ctx) expected = str(dataset_dir) assert uri == expected @@ -218,3 +209,174 @@ def test_extra_registry_contains_songliao_event(self): spec = HDS_DATASETS["songliao_event"] assert spec["reader"] == "floodevent" assert spec["path"] == "projects/songliao/event" + + +class TestWithHdsExtras: + """Verify _with_hds_extras injects HDS datasets and aliases correctly.""" + + def test_none_ctx_returns_hds_extras(self): + """None input returns context with HDS registry and aliases.""" + from hydrodatasource.configs.data_resolver import ( + HDS_DATASETS, + _HDS_READER_ALIASES, + _with_hds_extras, + ) + + ctx = _with_hds_extras(None) + assert HDS_DATASETS in ctx.extra_registry_dicts + assert ctx.extra_reader_aliases == _HDS_READER_ALIASES + + def test_provided_ctx_is_merged(self): + """Caller extras are preserved; HDS extras are appended on top.""" + from hydrodatasource.configs.data_resolver import ( + HDS_DATASETS, + _HDS_READER_ALIASES, + _with_hds_extras, + ) + + caller_dict = {"my_ds": {"reader": "my_reader", "path": "my/path"}} + caller_alias = {"my_reader": {"module": "m", "class": "C", "category": "t"}} + base_ctx = ResolverContext( + extra_registry_dicts=[caller_dict], + extra_reader_aliases=caller_alias, + ) + + merged = _with_hds_extras(base_ctx) + # Caller dict is first; HDS dict is last (override). + assert caller_dict in merged.extra_registry_dicts + assert HDS_DATASETS in merged.extra_registry_dicts + assert merged.extra_registry_dicts.index( + HDS_DATASETS + ) > merged.extra_registry_dicts.index(caller_dict) + # HDS aliases override caller aliases on conflict. + for key, spec in _HDS_READER_ALIASES.items(): + assert merged.extra_reader_aliases[key] == spec + + +class TestOpenDataset: + """Tests for hydrodatasource.configs.data_resolver.open_dataset.""" + + @pytest.fixture + def fake_flood_ctx(self, tmp_path): + """Patch FloodEventDatasource so open_dataset can be tested without real data.""" + dataset_dir = tmp_path / "projects" / "songliao" / "event" + dataset_dir.mkdir(parents=True) + + constructed = {} + + class FakeFloodEvent: + def __init__(self, uri, **kwargs): + constructed["uri"] = uri + constructed["kwargs"] = kwargs + + mod = types.ModuleType("hydrodatasource.reader.floodevent") + mod.FloodEventDatasource = FakeFloodEvent + sys.modules["hydrodatasource.reader.floodevent"] = mod + + ctx = ResolverContext( + storage={"local": {"root": str(tmp_path)}}, + ) + yield ctx, FakeFloodEvent, constructed, dataset_dir + + del sys.modules["hydrodatasource.reader.floodevent"] + + def test_open_dataset_songliao_event(self, fake_flood_ctx): + """open_dataset('songliao_event') resolves reader via registry, not alias key. + + This is the core regression test: 'songliao_event' -> reader 'floodevent' + (dataset_id != reader alias), which would KeyError without the registry lookup. + """ + from hydrodatasource.configs.data_resolver import open_dataset + + ctx, FakeFloodEvent, constructed, dataset_dir = fake_flood_ctx + result = open_dataset("songliao_event", ctx=ctx) + + assert isinstance(result, FakeFloodEvent) + assert constructed["uri"] == str(dataset_dir) + + def test_open_dataset_unknown_id_raises(self, tmp_path): + """Unknown dataset id raises DatasetResolutionError.""" + from hydrodatasource.configs.data_resolver import ( + DatasetResolutionError, + open_dataset, + ) + + ctx = ResolverContext(storage={"local": {"root": str(tmp_path)}}) + with pytest.raises(DatasetResolutionError): + open_dataset("nonexistent_xyz_hds", ctx=ctx) + + def test_open_dataset_exported_from_package(self): + """open_dataset is importable from the top-level hydrodatasource package.""" + from hydrodatasource import open_dataset as pkg_open + from hydrodatasource.configs.data_resolver import open_dataset + + assert pkg_open is open_dataset + + def test_open_dataset_forwards_kwargs(self, tmp_path): + """Extra kwargs are forwarded to the reader constructor.""" + dataset_dir = tmp_path / "my_dataset" + dataset_dir.mkdir() + + received = {} + + class KwargReader: + def __init__(self, uri, time_unit=None, **kwargs): + received["time_unit"] = time_unit + + mod = types.ModuleType("_hds_kwarg_reader_mod") + mod.KwargReader = KwargReader + sys.modules["_hds_kwarg_reader_mod"] = mod + + from hydrodatasource.configs.data_resolver import open_dataset + + ctx = ResolverContext( + storage={"local": {"root": str(tmp_path)}}, + extra_registry_dicts=[ + {"kwarg_hds_ds": {"reader": "kwarg_hds_reader", "path": "my_dataset"}} + ], + extra_reader_aliases={ + "kwarg_hds_reader": { + "module": "_hds_kwarg_reader_mod", + "class": "KwargReader", + "category": "test", + } + }, + ) + + open_dataset("kwarg_hds_ds", ctx=ctx, time_unit=["1D"]) + assert received["time_unit"] == ["1D"] + + del sys.modules["_hds_kwarg_reader_mod"] + + def test_open_dataset_cloud_s3_uri(self, tmp_path): + """open_dataset passes an S3 URI to the reader for cloud source.""" + received = {} + + class S3Reader: + def __init__(self, uri, **kwargs): + received["uri"] = uri + + mod = types.ModuleType("_hds_s3_reader_mod") + mod.S3Reader = S3Reader + sys.modules["_hds_s3_reader_mod"] = mod + + from hydrodatasource.configs.data_resolver import open_dataset + + ctx = ResolverContext( + storage={"s3": {"bucket": "hydro-bucket", "prefix": "hds"}}, + extra_registry_dicts=[ + {"s3_hds_ds": {"reader": "s3_hds_reader", "path": "cloud/path"}} + ], + extra_reader_aliases={ + "s3_hds_reader": { + "module": "_hds_s3_reader_mod", + "class": "S3Reader", + "category": "test", + } + }, + ) + + open_dataset("s3_hds_ds", source="cloud", ctx=ctx) + assert received["uri"] == "s3://hydro-bucket/hds/cloud/path" + + del sys.modules["_hds_s3_reader_mod"] diff --git a/tests/test_package_exports.py b/tests/test_package_exports.py index 0d29654..50877c8 100644 --- a/tests/test_package_exports.py +++ b/tests/test_package_exports.py @@ -59,9 +59,16 @@ def test_hydrodatasource_aliases_total_count(self): ) def test_merged_total_count(self): - """Total merged aliases: 11 hydrodatasource + 33 hydrodataset = 44.""" + """Total merged aliases: 11 hydrodatasource + 26 hydrodataset = 37.""" from hydrodatasource import READER_ALIASES - assert len(READER_ALIASES) == 44, ( - f"Expected 44 total aliases (11 HDS + 33 HD), got {len(READER_ALIASES)}" + assert len(READER_ALIASES) == 37, ( + f"Expected 37 total aliases (11 HDS + 26 HD), got {len(READER_ALIASES)}" ) + + def test_resolver_context_importable_from_package(self): + """ResolverContext is importable from hydrodatasource.""" + from hydrodatasource import ResolverContext + from dataclasses import is_dataclass + + assert is_dataclass(ResolverContext) From 436ac8051e455012ff3a2bb0e78cc1a57e090fc6 Mon Sep 17 00:00:00 2001 From: zym0123 <2689874960@qq.com> Date: Sun, 5 Jul 2026 11:48:52 +0800 Subject: [PATCH 25/34] feat: support NetCDF attributes and make basins.shp optional Read attributes from either attributes.csv (preferred, backward compatible) or a NetCDF file in the attributes/ dir, unifying csv/nc and local/minio via a single _read_attributes_df() helper. basins.shp becomes optional in cache_attributes_xrdataset: fall back to the 'area' column from attributes when no shape file is present. Co-Authored-By: Claude Opus 4.8 --- hydrodatasource/reader/data_source.py | 70 +++++++++++++++++---------- 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/hydrodatasource/reader/data_source.py b/hydrodatasource/reader/data_source.py index d22d6e6..9dd81bc 100644 --- a/hydrodatasource/reader/data_source.py +++ b/hydrodatasource/reader/data_source.py @@ -120,7 +120,7 @@ def set_data_source_describe(self): pattern = os.path.join(ts_dir, "*_units_info.json") unit_files = glob.glob(pattern) attr_dir = os.path.join(data_root_dir, "attributes") - attr_file = os.path.join(attr_dir, "attributes.csv") + attr_file = self._where_attr_file(attr_dir) shape_dir = os.path.join(data_root_dir, "shapes") return collections.OrderedDict( @@ -147,9 +147,34 @@ def _where_ts_dir(self, ts_dir): ] ) + def _where_attr_file(self, attr_dir): + """Locate the attributes file: prefer attributes.csv for backward + compatibility, otherwise fall back to a NetCDF file in the dir.""" + attr_csv = os.path.join(attr_dir, "attributes.csv") + if "s3://" in attr_dir: + return attr_csv + if os.path.exists(attr_csv): + return attr_csv + nc_files = sorted(glob.glob(os.path.join(attr_dir, "*.nc"))) + return nc_files[0] if nc_files else attr_csv + + def _read_attributes_df(self): + """Read the attributes file (csv or nc) into a DataFrame that has a + 'basin_id' column. Delegates to access_fs so that csv/nc and + local/minio are all handled uniformly.""" + attr_file = self.data_source_description["ATTR_FILE"] + obj = access_fs.spec_path(attr_file, head=self.head) + if isinstance(obj, xr.Dataset): + df = obj.to_dataframe().reset_index() + if "basin_id" not in df.columns and "basin" in df.columns: + df = df.rename(columns={"basin": "basin_id"}) + else: + df = obj + df["basin_id"] = df["basin_id"].astype(str) + return df + def read_site_info(self): - camels_file = self.data_source_description["ATTR_FILE"] - attrs = access_fs.spec_path(camels_file, head=self.head) + attrs = self._read_attributes_df() return attrs[["basin_id", "area"]] def read_object_ids(self, object_params=None) -> np.array: @@ -359,12 +384,7 @@ def read_attributes( self, object_ids=None, constant_cols=None, **kwargs ) -> np.array: """2d data (site_num * var_num), non-time-series data""" - attr_file = self.data_source_description["ATTR_FILE"] - if "s3://" in attr_file: - with conf.FS.open(attr_file, mode="rb") as f: - attrs = pd.read_csv(f, dtype={"basin_id": str}) - else: - attrs = pd.read_csv(attr_file, dtype={"basin_id": str}) + attrs = self._read_attributes_df() if object_ids is None: if constant_cols is None: return attrs @@ -380,12 +400,7 @@ def read_attributes( def get_attributes_cols(self) -> np.array: """the constant cols in this data_source""" - attr_file = self.data_source_description["ATTR_FILE"] - if "s3://" in attr_file: - with conf.FS.open(attr_file, mode="rb") as f: - attrs = pd.read_csv(f, dtype={"basin_id": str}) - else: - attrs = pd.read_csv(attr_file, dtype={"basin_id": str}) + attrs = self._read_attributes_df() attr_units = attrs.columns.values return self._check_vars_in_unitsinfo(attr_units) @@ -479,20 +494,23 @@ def cache_attributes_xrdataset(self): # NOTICE: although it seems that we don't use pint_xarray, we have to import this package import pint_xarray # noqa: F401 + df_attr = self.read_attributes() + df_attr.set_index("basin_id", inplace=True) + + # basins.shp is optional: when present it provides shp_area; otherwise + # we rely on the 'area' column already in the attributes file. shape_dir = os.path.join( self.data_source_description["SHAPE_DIR"], "basins.shp" ) - if "s3://" in shape_dir: - with conf.FS.open(shape_dir, mode="rb") as f: - shape = gpd.read_file(f) - else: - shape = gpd.read_file(shape_dir) - df_area = cal_area_from_shp(shape) # calculate the area from shape file - df_area.set_index("basin_id", inplace=True) - - df_attr = self.read_attributes() - df_attr.set_index("basin_id", inplace=True) - df_attr = df_attr.join(df_area) + if "s3://" in shape_dir or os.path.exists(shape_dir): + if "s3://" in shape_dir: + with conf.FS.open(shape_dir, mode="rb") as f: + shape = gpd.read_file(f) + else: + shape = gpd.read_file(shape_dir) + df_area = cal_area_from_shp(shape) # area from shape file + df_area.set_index("basin_id", inplace=True) + df_attr = df_attr.join(df_area) # Mapping provided units to the variables in the datasets # For attributes, all the variables' units are same in all unit_info files # hence, we just chose the first one From 7397166325e91322cb303e37ab71b37502fad3d8 Mon Sep 17 00:00:00 2001 From: zym0123 <2689874960@qq.com> Date: Sun, 5 Jul 2026 17:30:20 +0800 Subject: [PATCH 26/34] fix: create cache dir if missing to avoid NetCDF write failure On a fresh machine the resolved CACHE_DIR (e.g. ~/.cache/hydrodataset) may not exist yet; h5netcdf/os.listdir do not create it, so caching attributes/ timeseries raised FileNotFoundError. Create the dir once at CACHE_DIR resolution with makedirs(exist_ok=True). Co-Authored-By: Claude Opus 4.8 --- hydrodatasource/configs/config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hydrodatasource/configs/config.py b/hydrodatasource/configs/config.py index 6282ae1..c531834 100644 --- a/hydrodatasource/configs/config.py +++ b/hydrodatasource/configs/config.py @@ -87,6 +87,9 @@ def _init_settings() -> None: _lazy["SETTING"] = setting _lazy["LOCAL_ROOT"] = root _lazy["CACHE_DIR"] = str(_hd_get_cache_dir()) + # Ensure the cache dir exists: downstream code writes NetCDF caches and + # lists this dir directly, and h5netcdf/os.listdir do not create it. + os.makedirs(_lazy["CACHE_DIR"], exist_ok=True) # Initialize S3FS from storage.s3 credentials s3_cfg = setting.get("storage", {}).get("s3", {}) From 0fee67edd23fdf7d0d12529d0d7d5fb8d2919825 Mon Sep 17 00:00:00 2001 From: owen Date: Sat, 15 Aug 2026 15:15:52 +0800 Subject: [PATCH 27/34] fix: harden data interface and adapt to hydroutils 0.2.0 Data source hardening: - require uri in HydroData, reject legacy data_path/dataset_name/data_folder kwargs - preserve zero-padded basin_id via dtype passthrough in access_fs - fix streamflow_unit mapping '1MS'->'1M'; add GLOBAL_DIR to LongTermDataset - floodevent: filter on 'basin' coord (read side), not gage_id - data_resolver: source=None falls back to storage.default_source - config: warn when hydro_setting.yml lacks a 'storage' section - remove dead scripts/example_tg_usage.py hydroutils 0.2.0 migration (hydro_log module removed): - replace @hydro_logger decorator with stdlib logging in gages, rainfall_reader, rainfall_cleaner, rsvr_inflow_cleaner; fixes latent .info() call on the decorator function in gages.prepare_usgs_data - bump dependency to hydroutils>=0.2.0 Tests: - add regression tests (test_fix_*, test_hydroutils_compat) - fix CACHE_DIR isolation and sys.modules teardown pollution - de-hardcode package export count --- .gitignore | 3 + hydrodatasource/cleaner/rainfall_cleaner.py | 7 +- .../cleaner/rsvr_inflow_cleaner.py | 6 +- hydrodatasource/configs/config.py | 6 + hydrodatasource/configs/data_resolver.py | 14 +- hydrodatasource/reader/access_fs.py | 21 +- hydrodatasource/reader/data_source.py | 22 +- hydrodatasource/reader/floodevent.py | 4 +- hydrodatasource/reader/gages.py | 6 +- hydrodatasource/reader/rainfall_reader.py | 3 - pyproject.toml | 2 +- scripts/example_tg_usage.py | 158 ------------ tests/test_data_resolver.py | 15 +- tests/test_fix_data_source.py | 139 +++++++++++ tests/test_fix_floodevent.py | 231 ++++++++++++++++++ tests/test_fix_resolver_config.py | 78 ++++++ tests/test_hydroutils_compat.py | 110 +++++++++ tests/test_package_exports.py | 19 +- tests/test_tg_hydro_datasource.py | 34 ++- uv.lock | 178 ++++---------- 20 files changed, 721 insertions(+), 335 deletions(-) delete mode 100644 scripts/example_tg_usage.py create mode 100644 tests/test_fix_data_source.py create mode 100644 tests/test_fix_floodevent.py create mode 100644 tests/test_fix_resolver_config.py create mode 100644 tests/test_hydroutils_compat.py diff --git a/.gitignore b/.gitignore index 11e12db..ad365d6 100644 --- a/.gitignore +++ b/.gitignore @@ -174,3 +174,6 @@ GEMINI.md # Cursor code graph .codegraph/ + +# Specify / Speckit local tooling +.specify/ diff --git a/hydrodatasource/cleaner/rainfall_cleaner.py b/hydrodatasource/cleaner/rainfall_cleaner.py index 8eae6fc..c03d395 100644 --- a/hydrodatasource/cleaner/rainfall_cleaner.py +++ b/hydrodatasource/cleaner/rainfall_cleaner.py @@ -16,7 +16,6 @@ from shapely.geometry import Point from geopandas.tools import sjoin from tqdm import tqdm -from hydroutils.hydro_log import hydro_logger from hydrodatasource.processor.basin_mean_rainfall import ( calculate_thiesen_polygons, calculate_weighted_rainfall, @@ -24,8 +23,9 @@ from hydrodatasource.cleaner.cleaner import Cleaner -@hydro_logger class RainfallCleaner(Cleaner): + logger = logging.getLogger(__name__) + def __init__(self, data_folder, output_folder): """All files to be cleaned are in the data_dir @@ -548,8 +548,9 @@ def rainfall_clean(self, basin_id, **kwargs): ) -@hydro_logger class RainfallAnalyzer: + logger = logging.getLogger(__name__) + def __init__( self, data_folder=None, diff --git a/hydrodatasource/cleaner/rsvr_inflow_cleaner.py b/hydrodatasource/cleaner/rsvr_inflow_cleaner.py index 862c694..b22dbc2 100644 --- a/hydrodatasource/cleaner/rsvr_inflow_cleaner.py +++ b/hydrodatasource/cleaner/rsvr_inflow_cleaner.py @@ -16,13 +16,15 @@ from tqdm import tqdm from scipy.optimize import curve_fit -from hydroutils.hydro_log import hydro_logger +import logging + from hydrodatasource.configs.table_name import RSVR_TS_TABLE_COLS from hydrodatasource.cleaner.cleaner import Cleaner -@hydro_logger class ReservoirInflowBacktrack(Cleaner): + logger = logging.getLogger(__name__) + def __init__(self, data_folder, output_folder): """ Back-calculating inflow of reservior diff --git a/hydrodatasource/configs/config.py b/hydrodatasource/configs/config.py index c531834..3bd6beb 100644 --- a/hydrodatasource/configs/config.py +++ b/hydrodatasource/configs/config.py @@ -81,6 +81,12 @@ def _init_settings() -> None: stacklevel=2, ) setting = {"storage": {"local": {"root": default_root}}} + elif "storage" not in setting: + warnings.warn( + f"hydro_setting.yml is non-empty but missing a 'storage' section; " + f"falling back to default data root: {default_root}", + stacklevel=2, + ) root = setting.get("storage", {}).get("local", {}).get("root", default_root) diff --git a/hydrodatasource/configs/data_resolver.py b/hydrodatasource/configs/data_resolver.py index 736c0b9..0d4ca2d 100644 --- a/hydrodatasource/configs/data_resolver.py +++ b/hydrodatasource/configs/data_resolver.py @@ -164,7 +164,7 @@ def _with_hds_extras(ctx: Optional[ResolverContext]) -> ResolverContext: def resolve_data_path( dataset_id: str, *, - source: Literal["local", "cloud"] = "local", + source: Optional[Literal["local", "cloud"]] = None, ctx: Optional[ResolverContext] = None, ) -> str: """Resolve a dataset id to an absolute data path (URI). @@ -177,8 +177,9 @@ def resolve_data_path( ---------- dataset_id : str Dataset identifier from the registry (e.g. 'camels_us', 'songliao_event'). - source : str - Storage backend: 'local' or 'cloud'. + source : str, optional + Storage backend: 'local' or 'cloud'. When None (default), falls back + to ``storage.default_source`` from the settings. ctx : ResolverContext, optional Resolution context bundling project_root, storage config, registry overrides, and extra aliases. When None (default), a new context @@ -210,7 +211,7 @@ def resolve_data_path( def open_dataset( dataset_id: str, *, - source: Literal["local", "cloud"] = "local", + source: Optional[Literal["local", "cloud"]] = None, ctx: Optional[ResolverContext] = None, **reader_kwargs: Any, ): @@ -226,8 +227,9 @@ def open_dataset( ---------- dataset_id : str Dataset identifier from the registry (e.g. 'camels_us', 'songliao_event'). - source : str - Storage backend: 'local' or 'cloud'. + source : str, optional + Storage backend: 'local' or 'cloud'. When None (default), falls back + to ``storage.default_source`` from the settings. ctx : ResolverContext, optional Resolution context. When None, HDS defaults are used automatically. **reader_kwargs diff --git a/hydrodatasource/reader/access_fs.py b/hydrodatasource/reader/access_fs.py index d35ecd2..171ad55 100644 --- a/hydrodatasource/reader/access_fs.py +++ b/hydrodatasource/reader/access_fs.py @@ -13,7 +13,7 @@ import hydrodatasource.configs.config as conf -def spec_path(url_path: str, head="local", need_cache=False, is_dir=False): +def spec_path(url_path: str, head="local", need_cache=False, is_dir=False, dtype=None): """Access the file system to get data of specific path, file or directory. Parameters @@ -26,6 +26,9 @@ def spec_path(url_path: str, head="local", need_cache=False, is_dir=False): _description_, by default False is_dir : bool, optional file or directory, by default False, means file + dtype : dict, optional + Column type overrides forwarded to pandas when reading tabular + files (e.g. ``{"basin_id": str}`` to preserve zero-padded ids). Returns ------- @@ -41,11 +44,14 @@ def spec_path(url_path: str, head="local", need_cache=False, is_dir=False): """ if is_dir is False: if head == "local": - ret_data = read_valid_data(url_path, need_cache=need_cache) + ret_data = read_valid_data(url_path, need_cache=need_cache, dtype=dtype) elif head == "minio": url_path = f"s3://{url_path}" ret_data = read_valid_data( - url_path, storage_option=conf.MINIO_PARAM, need_cache=need_cache + url_path, + storage_option=conf.MINIO_PARAM, + need_cache=need_cache, + dtype=dtype, ) else: raise ValueError("head should be 'local' or 'minio'") @@ -74,7 +80,7 @@ def spec_path(url_path: str, head="local", need_cache=False, is_dir=False): return ret_data -def read_valid_data(obj: str, storage_option=None, need_cache=False, need_refer=False): +def read_valid_data(obj: str, storage_option=None, need_cache=False, need_refer=False, dtype=None): """ Read valid data from different file types. See https://intake.readthedocs.io/en/latest/plugin-directory.html @@ -90,6 +96,9 @@ def read_valid_data(obj: str, storage_option=None, need_cache=False, need_refer= See hydrodatasource.configs.config.MINIO_PARAM to get reference need_cache (bool, optional) Whether to cache the data. Defaults to False. + dtype (dict, optional) + Column type overrides forwarded to pandas when reading tabular + files (e.g. ``{"basin_id": str}``). Returns: ------------ @@ -105,7 +114,9 @@ def read_valid_data(obj: str, storage_option=None, need_cache=False, need_refer= else: ext_name = obj.split(".")[-1] if ext_name == "csv": - data_obj = pd.read_csv(obj, storage_options=storage_option) + data_obj = pd.read_csv( + obj, storage_options=storage_option, dtype=dtype + ) if need_cache & (storage_option is not None): data_obj.to_csv( path_or_buf=os.path.join(conf.LOCAL_ROOT, cache_name) diff --git a/hydrodatasource/reader/data_source.py b/hydrodatasource/reader/data_source.py index 9dd81bc..2bf91b4 100644 --- a/hydrodatasource/reader/data_source.py +++ b/hydrodatasource/reader/data_source.py @@ -44,6 +44,11 @@ class HydroData(ABC): """ def __init__(self, uri): + if uri is None: + raise ValueError( + "uri must be provided: pass an absolute path or s3:// URI " + "pointing to the data directory." + ) self.data_source_dir = str(uri) self.dataset_name = Path(str(uri)).name @@ -84,6 +89,14 @@ def __init__( kwargs : dict, optional Additional keyword arguments, by default None. """ + legacy = [ + k for k in ("data_path", "dataset_name", "data_folder") if k in kwargs + ] + if legacy: + raise ValueError( + f"Unsupported legacy argument(s) {legacy}; they were replaced by " + "'uri='. Pass the absolute path or s3:// URI as uri=." + ) if time_unit and any( unit not in ["1h", "3h", "1D", "8D", "1M"] for unit in time_unit ): @@ -105,7 +118,7 @@ def __init__( @property def streamflow_unit(self): - unit_mapping = {"1h": "mm/h", "3h": "mm/3h", "1D": "mm/d", "1MS": "mm/M"} + unit_mapping = {"1h": "mm/h", "3h": "mm/3h", "1D": "mm/d", "1M": "mm/M"} return {unit: unit_mapping[unit] for unit in self.time_unit} def get_name(self): @@ -163,7 +176,9 @@ def _read_attributes_df(self): 'basin_id' column. Delegates to access_fs so that csv/nc and local/minio are all handled uniformly.""" attr_file = self.data_source_description["ATTR_FILE"] - obj = access_fs.spec_path(attr_file, head=self.head) + obj = access_fs.spec_path( + attr_file, head=self.head, dtype={"basin_id": str} + ) if isinstance(obj, xr.Dataset): df = obj.to_dataframe().reset_index() if "basin_id" not in df.columns and "basin" in df.columns: @@ -892,6 +907,9 @@ def set_data_source_describe(self): the_dict["ATTR_FILE"] = os.path.join( self.data_source_dir, "attributes", "hydroatlas_attributes.csv" ) + the_dict["GLOBAL_DIR"] = os.path.join( + self.data_source_dir, "attributes" + ) the_dict["TELECONNECTIONS_DIR"] = self._where_ts_dir( os.path.join(self.data_source_dir, "teleconnections") ) diff --git a/hydrodatasource/reader/floodevent.py b/hydrodatasource/reader/floodevent.py index 2d7ce47..c010026 100644 --- a/hydrodatasource/reader/floodevent.py +++ b/hydrodatasource/reader/floodevent.py @@ -923,10 +923,10 @@ def read_ts_xrdataset_augmented( available_stations = [ station for station in gage_id_lst - if station in ds.coords.get("gage_id", []) + if station in ds.coords.get("basin", []) ] if available_stations: - ds = ds.sel(gage_id=available_stations) + ds = ds.sel(basin=available_stations) # 应用时间范围过滤 if t_range is not None and len(t_range) >= 2: diff --git a/hydrodatasource/reader/gages.py b/hydrodatasource/reader/gages.py index d5faaee..7d1261c 100644 --- a/hydrodatasource/reader/gages.py +++ b/hydrodatasource/reader/gages.py @@ -20,11 +20,13 @@ import pytz from pandas.core.dtypes.common import is_string_dtype, is_numeric_dtype from tqdm import tqdm -from hydroutils import hydro_logger, hydro_file, hydro_stat +from hydroutils import hydro_file, hydro_stat import hydrodatasource.configs.config as conf from hydrodatasource.configs.config import CACHE_DIR from hydrodatasource.reader.data_source import HydroData +logger = logging.getLogger(__name__) + class Gages(HydroData): def __init__(self, uri): @@ -590,7 +592,7 @@ def read_site_info(self): return pd.DataFrame(out) def prepare_usgs_data(self): - hydro_logger.info( + logger.info( "NOT all data_source could be downloaded from website directly!" ) data_source_description = self.data_source_description diff --git a/hydrodatasource/reader/rainfall_reader.py b/hydrodatasource/reader/rainfall_reader.py index 54fcee2..5b4806a 100644 --- a/hydrodatasource/reader/rainfall_reader.py +++ b/hydrodatasource/reader/rainfall_reader.py @@ -15,8 +15,6 @@ from tqdm import tqdm import xarray as xr -from hydroutils.hydro_log import hydro_logger - from hydrodatasource.reader.data_source import HydroData @@ -80,7 +78,6 @@ def filter_out_rows(df1, df2, key_columns): return filtered_df -@hydro_logger class RainfallReader(HydroData): def __init__(self, uri): """A reader for basin rainfall data. diff --git a/pyproject.toml b/pyproject.toml index 5d6e1af..92a05ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,7 @@ dependencies = [ "pygeohydro", "pykalman", "xarray", - "hydroutils>=0.1.0", + "hydroutils>=0.2.0", "hydrodataset>=0.2.0", "hydrotopo", "aqua-fetch[all]", diff --git a/scripts/example_tg_usage.py b/scripts/example_tg_usage.py deleted file mode 100644 index 8dbe39d..0000000 --- a/scripts/example_tg_usage.py +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -简化后的TgHydroDatasource类使用示例 - -该示例展示了如何使用简化后的TgHydroDatasource类进行数据读取， -不涉及torch转换，保持数据的原始xarray格式。 -""" - -import os -import sys -from pathlib import Path - -# 添加项目路径 -project_root = Path(__file__).parent -sys.path.insert(0, str(project_root)) - -from hydrodatasource.reader.data_source import TgHydroDatasource - - -def example_usage(): - """展示TgHydroDatasource的基本使用方法""" - - # 假设数据路径 - data_path = "/path/to/your/tg_hydro_data" - - # 初始化TgHydroDatasource - tg_datasource = TgHydroDatasource( - data_path=data_path, time_unit=["1D"], dataset_name="example_dataset" - ) - - print(f"数据源名称: {tg_datasource.get_name()}") - print(f"数据路径: {tg_datasource.data_source_dir}") - - # 1. 读取可用的流域ID - available_basins = tg_datasource.read_object_ids() - print(f"可用流域数量: {len(available_basins)}") - print(f"前5个流域ID: {available_basins[:5]}") - - # 2. 读取时间序列数据和LSTM预测数据 - selected_basins = available_basins[:3] # 选择前3个流域 - time_range = ["2020-01-01", "2020-12-31"] - variables = ["total_precipitation_hourly", "streamflow"] - - timeseries_data, lstm_data = tg_datasource.read_timeseries_with_lstm( - object_ids=selected_basins, t_range=time_range, var_lst=variables - ) - - print(f"\n时间序列数据:") - print(f" 数据类型: {type(timeseries_data)}") - print(f" 时间单位: {list(timeseries_data.keys())}") - for time_unit, data in timeseries_data.items(): - print(f" {time_unit} 数据形状: {data.dims}") - print(f" {time_unit} 变量: {list(data.data_vars)}") - - print(f"\nLSTM预测数据:") - print(f" 数据类型: {type(lstm_data)}") - print(f" 数据形状: {lstm_data.dims}") - print(f" 变量: {list(lstm_data.data_vars)}") - - # 3. 读取属性数据 - selected_attrs = ["ele_mt_sav", "area", "p_mean"] - attr_data = tg_datasource.read_node_attributes( - object_ids=selected_basins, selected_attrs=selected_attrs - ) - - print(f"\n属性数据:") - print(f" 数据类型: {type(attr_data)}") - print(f" 数据形状: {attr_data.dims}") - print(f" 变量: {list(attr_data.data_vars)}") - - # 4. 生成图网络结构 - basin_id = "test_basin" # 根据实际情况修改 - try: - dg, edges, node_mapping, valid_object_ids = tg_datasource.gen_nx_graph( - basin_id=basin_id, object_ids=selected_basins - ) - - print(f"\n图网络结构:") - print(f" 节点数量: {len(dg.nodes)}") - print(f" 边数量: {len(edges)}") - print(f" 节点映射: {node_mapping}") - print(f" 有效对象ID: {valid_object_ids}") - - # 5. 获取上游节点 - if valid_object_ids: - target_node = valid_object_ids[0] - upstream_nodes = tg_datasource.get_upstream_nodes( - basin_id=basin_id, target_node=target_node, object_ids=selected_basins - ) - print(f" {target_node} 的上游节点索引: {upstream_nodes}") - - except ValueError as e: - print(f"\n图网络结构错误: {e}") - print("请检查graph_dict.json文件是否存在且包含正确的流域ID") - - # 6. 使用父类方法读取数据（展示继承功能） - print(f"\n使用父类方法读取数据:") - - # 读取时间序列数据 - ts_data = tg_datasource.read_ts_xrdataset( - gage_id_lst=selected_basins, - t_range=time_range, - var_lst=["streamflow"], - time_units=["1D"], - ) - print(f" 时间序列数据 (父类方法): {type(ts_data)}") - - # 读取属性数据 - attr_data_parent = tg_datasource.read_attr_xrdataset(var_lst=["area"]) - print(f" 属性数据 (父类方法): {type(attr_data_parent)}") - - -def data_processing_example(): - """展示如何处理读取的数据""" - - print("\n" + "=" * 50) - print("数据处理示例") - print("=" * 50) - - # 这里展示如何处理从TgHydroDatasource读取的xarray数据 - # 用户可以根据需要进行各种数据处理和分析 - - print("1. 数据筛选和切片") - print(" - 使用 .sel() 方法按坐标筛选") - print(" - 使用 .isel() 方法按索引筛选") - print(" - 使用 .where() 方法按条件筛选") - - print("\n2. 数据转换") - print(" - 使用 .values 获取numpy数组") - print(" - 使用 .to_pandas() 转换为pandas") - print(" - 使用 .to_netcdf() 保存为NetCDF文件") - - print("\n3. 数据分析") - print(" - 使用 .mean(), .std(), .max(), .min() 等统计方法") - print(" - 使用 .groupby() 进行分组分析") - print(" - 使用 .resample() 进行时间重采样") - - print("\n4. 如果需要torch张量，可以手动转换:") - print(" import torch") - print(" tensor = torch.from_numpy(data.values).float()") - - -if __name__ == "__main__": - print("TgHydroDatasource 简化版使用示例") - print("=" * 50) - - try: - example_usage() - data_processing_example() - - print("\n" + "=" * 50) - print("示例完成！") - print("=" * 50) - - except Exception as e: - print(f"运行示例时出错: {e}") - print("请确保数据路径正确且数据文件存在") diff --git a/tests/test_data_resolver.py b/tests/test_data_resolver.py index c36fd37..be7278a 100644 --- a/tests/test_data_resolver.py +++ b/tests/test_data_resolver.py @@ -10,6 +10,8 @@ from hydrodataset.configs.data_resolver import ResolverContext +import hydrodatasource.reader.floodevent as _real_floodevent_module + class TestReaderAliases: """Verify READER_ALIASES covers all hydrodatasource reader classes.""" @@ -273,12 +275,13 @@ def __init__(self, uri, **kwargs): mod.FloodEventDatasource = FakeFloodEvent sys.modules["hydrodatasource.reader.floodevent"] = mod - ctx = ResolverContext( - storage={"local": {"root": str(tmp_path)}}, - ) - yield ctx, FakeFloodEvent, constructed, dataset_dir - - del sys.modules["hydrodatasource.reader.floodevent"] + try: + ctx = ResolverContext( + storage={"local": {"root": str(tmp_path)}}, + ) + yield ctx, FakeFloodEvent, constructed, dataset_dir + finally: + sys.modules["hydrodatasource.reader.floodevent"] = _real_floodevent_module def test_open_dataset_songliao_event(self, fake_flood_ctx): """open_dataset('songliao_event') resolves reader via registry, not alias key. diff --git a/tests/test_fix_data_source.py b/tests/test_fix_data_source.py new file mode 100644 index 0000000..9df2dfb --- /dev/null +++ b/tests/test_fix_data_source.py @@ -0,0 +1,139 @@ +"""Regression tests for three defects in hydrodatasource.reader.data_source. + +Each test is written to FAIL (RED) against the current, buggy implementation +and to PASS once the corresponding defect is fixed (GREEN): + +- Defect A: ``uri=None`` silently generates a ``"None"`` path, and the legacy + kwargs ``data_path`` / ``dataset_name`` / ``data_folder`` are silently + swallowed by ``**kwargs`` instead of being rejected with a ``ValueError``. +- Defect B: ``basin_id`` leading zeros are lost when reading + ``attributes/attributes.csv`` (``pd.read_csv`` infers ``int64`` for pure + numeric IDs, so ``"01015500"`` becomes ``"1015500"``). +- Defect C: the ``streamflow_unit`` property maps ``"1MS"`` while the + ``time_unit`` whitelist uses ``"1M"``, so accessing ``streamflow_unit`` for + the supported unit ``"1M"`` raises ``KeyError``. + +The tests are self-contained: they build a minimal Caravan-style dataset +directory under ``tmp_path`` and never touch real data. +""" + +import pytest + +from hydrodatasource.reader.data_source import SelfMadeHydroDataset + + +@pytest.fixture +def selfmade_dataset_dir(tmp_path): + """Build a minimal Caravan-style dataset directory. + + Structure:: + + /selfmade_ds/ + ├── attributes/ + │ └── attributes.csv # basin_id,area (zero-padded IDs) + └── timeseries/ + ├── 1D/ # empty time-unit dir (for _where_ts_dir) + └── 1M/ # empty time-unit dir (for _where_ts_dir) + + The ``timeseries`` subdirectories are required because + ``set_data_source_describe`` lists them via ``_where_ts_dir``. + """ + data_dir = tmp_path / "selfmade_ds" + + attrs_dir = data_dir / "attributes" + attrs_dir.mkdir(parents=True) + (attrs_dir / "attributes.csv").write_text( + "basin_id,area\n01015500,12345\n01020500,67890\n", + encoding="utf-8", + ) + + ts_dir = data_dir / "timeseries" + (ts_dir / "1D").mkdir(parents=True) + (ts_dir / "1M").mkdir(parents=True) + + return data_dir + + +# ── Defect A: uri=None silently generates a "None" path; legacy kwargs swallowed ── + + +class TestConstructorRequiresUri: + """Construction without ``uri=`` must raise ``ValueError``. + + Current bug: ``uri=None`` is passed through to ``HydroData.__init__`` as + ``str(None) == "None"``, and legacy kwargs (``data_path``, ``dataset_name``, + ``data_folder``) are swallowed by ``**kwargs``. The constructor then blows + up with a ``FileNotFoundError`` on the bogus ``"None/timeseries"`` path + instead of a clear ``ValueError``. + """ + + @pytest.mark.parametrize( + "kwargs", + [ + {}, # no arguments at all + {"data_path": "/some/data"}, + {"dataset_name": "my_dataset"}, + {"data_folder": "/some/data"}, + ], + ids=["no-args", "data_path", "dataset_name", "data_folder"], + ) + def test_missing_uri_raises_valueerror(self, kwargs): + """SelfMadeHydroDataset() without uri= raises ValueError.""" + with pytest.raises(ValueError): + SelfMadeHydroDataset(**kwargs) + + +# ── Defect B: basin_id leading zeros are lost when reading attributes.csv ── + + +class TestBasinIdLeadingZeros: + """``basin_id`` read from attributes.csv must keep zero padding. + + Current bug: ``_read_attributes_df`` reads the CSV via + ``access_fs.spec_path`` (``pd.read_csv`` without a ``dtype``), so the pure + numeric IDs ``01015500`` / ``01020500`` are inferred as ``int64`` and + ``.astype(str)`` turns them into ``"1015500"`` / ``"1020500"``. + """ + + def test_basin_id_preserves_leading_zeros(self, selfmade_dataset_dir): + """Zero-padded basin IDs survive the read as strings.""" + ds = SelfMadeHydroDataset( + uri=str(selfmade_dataset_dir), + time_unit=["1D"], + ) + + assert ds.camels_sites["basin_id"].tolist() == [ + "01015500", + "01020500", + ] + + +# ── Defect C: streamflow_unit mapping is missing the "1M" key ── + + +class TestStreamflowUnit: + """``streamflow_unit`` must support the ``"1M"`` time unit. + + Current bug: ``streamflow_unit`` maps the key ``"1MS"`` while the + ``time_unit`` whitelist (and therefore ``self.time_unit``) uses ``"1M"``, + so ``unit_mapping["1M"]`` raises ``KeyError``. + """ + + def test_streamflow_unit_1m_key(self, selfmade_dataset_dir): + """streamflow_unit maps the supported unit "1M" -> "mm/M".""" + ds = SelfMadeHydroDataset( + uri=str(selfmade_dataset_dir), + time_unit=["1M"], + ) + + assert ds.streamflow_unit == {"1M": "mm/M"} + + def test_streamflow_unit_covers_all_supported_units(self, selfmade_dataset_dir): + """streamflow_unit covers every supported time unit (1h/3h/1D/1M).""" + ds = SelfMadeHydroDataset( + uri=str(selfmade_dataset_dir), + time_unit=["1h", "3h", "1D", "1M"], + ) + + assert set(ds.streamflow_unit.keys()) == {"1h", "3h", "1D", "1M"} + assert ds.streamflow_unit["1M"] == "mm/M" diff --git a/tests/test_fix_floodevent.py b/tests/test_fix_floodevent.py new file mode 100644 index 0000000..724d6b4 --- /dev/null +++ b/tests/test_fix_floodevent.py @@ -0,0 +1,231 @@ +"""Regression tests for the floodevent augmented-timeseries dimension bug (C1). + +Root cause +---------- +The write/merge path of ``FloodEventDatasource`` stores the site dimension under +the name ``basin``: + + create_xarray_dataset_from_augdf(...) -> coords={"time": ..., "basin": [station_id]} + _process_augmented_files_by_paths(...) -> xr.concat(all_datasets, dim="basin") + +but ``read_ts_xrdataset_augmented`` filters on a ``gage_id`` coordinate that is +never written: + + available_stations = [ + station for station in gage_id_lst + if station in ds.coords.get("gage_id", []) + ] + +Because no ``gage_id`` coord exists, ``available_stations`` is always ``[]`` and +the filter is skipped — so the read returns EVERY basin in the cache file instead +of only the requested subset. + +Expected post-fix behaviour +--------------------------- +``read_ts_xrdataset_augmented(gage_id_lst=[...])`` must filter on the ``basin`` +dimension so the returned Dataset's ``basin`` coord contains exactly the +requested stations. + +These tests reproduce the bug (RED on the current code) and act as the +regression guard for the fix. +""" + +import numpy as np +import pandas as pd +import pytest +import xarray as xr + +import hydrodatasource.reader.floodevent as floodevent_module + + +# ── Fixtures ──────────────────────────────────────────────────────────────── + + +@pytest.fixture +def floodevent_datasource(tmp_path): + """Instantiate a real FloodEventDatasource on a minimal tmp dataset. + + FloodEventDatasource.__init__ validates a local filesystem layout with + ``required_subdirs = ["attributes", "timeseries"]``, so we create exactly + that structure (plus a ``3h`` timeseries sub-directory which + ``set_data_source_describe`` expects to enumerate). + """ + data_dir = tmp_path / "floodevent_test_dataset" + attrs_dir = data_dir / "attributes" + ts_dir = data_dir / "timeseries" / "3h" + attrs_dir.mkdir(parents=True) + ts_dir.mkdir(parents=True) + + pd.DataFrame( + { + "basin_id": ["01015500", "01015501", "01020500"], + "area": [100.0, 150.0, 200.0], + } + ).to_csv(attrs_dir / "attributes.csv", index=False) + + from hydrodatasource.reader.floodevent import FloodEventDatasource + + return FloodEventDatasource(uri=str(data_dir), time_unit=["3h"]) + + +@pytest.fixture +def augmented_cache_dir(tmp_path, monkeypatch): + """Redirect the floodevent module's CACHE_DIR to a temp directory. + + floodevent.py binds ``CACHE_DIR`` at import time via + ``from hydrodatasource.configs.config import CACHE_DIR``, so the module-level + attribute must be patched (not ``config.CACHE_DIR``). + """ + cache_dir = tmp_path / "cache" + cache_dir.mkdir(exist_ok=True) + monkeypatch.setattr(floodevent_module, "CACHE_DIR", str(cache_dir)) + return cache_dir + + +# ── Helpers ───────────────────────────────────────────────────────────────── + + +def _make_augmented_dataset(basins, n_time=8): + """Mimic the write-end output: one ``basin`` coord holding all sites. + + This matches the format produced by ``create_xarray_dataset_from_augdf`` + plus ``xr.concat(..., dim="basin")``. + """ + time = pd.date_range("2020-01-01", periods=n_time, freq="3h") + rng = np.random.default_rng(0) + n_basin = len(basins) + return xr.Dataset( + data_vars={ + "rain": (["basin", "time"], rng.random((n_basin, n_time))), + "inflow": (["basin", "time"], rng.random((n_basin, n_time)) * 100.0), + "flood_event": ( + ["basin", "time"], + np.zeros((n_basin, n_time), dtype=np.int32), + ), + "ES": (["basin", "time"], rng.random((n_basin, n_time))), + }, + coords={"time": time, "basin": basins}, + attrs={"description": "Augmented hydrological time series data"}, + ) + + +def _cache_name(datasource, time_unit, first_station, last_station): + """Recompute the augmented cache file name using the reader's own rule.""" + return ( + f"{datasource.dataset_name}_dataaugment_" + f"timeseries_{time_unit}_batch_{first_station}_{last_station}.nc" + ) + + +# ── RED tests: read side must filter on the ``basin`` dimension ───────────── + + +def test_read_augmented_returns_only_requested_basin( + floodevent_datasource, augmented_cache_dir +): + """Requesting one basin from a multi-basin cache must return only that basin. + + The cache file is found (name matches the reader's lookup for a single-station + request) but contains an extra basin. The current code skips the filter + (no ``gage_id`` coord) and returns both basins -> RED. + """ + ds = _make_augmented_dataset(basins=["01015500", "01020500"]) + cache_file = augmented_cache_dir / _cache_name( + floodevent_datasource, "3h", "01015500", "01015500" + ) + ds.to_netcdf(cache_file) + + result = floodevent_datasource.read_ts_xrdataset_augmented( + gage_id_lst=["01015500"], + time_unit="3h", + ) + + assert "3h" in result, "augmented read did not return the '3h' entry" + returned_basins = result["3h"]["basin"].values.tolist() + assert returned_basins == ["01015500"], ( + f"expected only ['01015500'], got {returned_basins} " + f"(filter looked at 'gage_id' instead of 'basin')" + ) + + +def test_read_augmented_subset_of_batch_returns_only_requested( + floodevent_datasource, augmented_cache_dir +): + """Requesting a sub-range of a batch file must not leak extra basins. + + A batch cache file covers a range of station IDs (first/last in the name) and + can contain intermediate basins. Requesting two of the three basins must + return exactly those two. The buggy filter returns all three -> RED. + """ + ds = _make_augmented_dataset( + basins=["01015500", "01015501", "01020500"] + ) + cache_file = augmented_cache_dir / _cache_name( + floodevent_datasource, "3h", "01015500", "01020500" + ) + ds.to_netcdf(cache_file) + + result = floodevent_datasource.read_ts_xrdataset_augmented( + gage_id_lst=["01015500", "01020500"], + time_unit="3h", + ) + + assert "3h" in result, "augmented read did not return the '3h' entry" + returned_basins = result["3h"]["basin"].values.tolist() + assert returned_basins == ["01015500", "01020500"], ( + f"expected ['01015500', '01020500'], got {returned_basins} " + f"(intermediate basin '01015501' should have been filtered out)" + ) + + +def test_read_augmented_filters_basin_with_t_range_and_var_lst( + floodevent_datasource, augmented_cache_dir +): + """The basin-filter bug persists even when t_range / var_lst are supplied.""" + ds = _make_augmented_dataset(basins=["01015500", "01020500"]) + cache_file = augmented_cache_dir / _cache_name( + floodevent_datasource, "3h", "01015500", "01015500" + ) + ds.to_netcdf(cache_file) + + result = floodevent_datasource.read_ts_xrdataset_augmented( + gage_id_lst=["01015500"], + t_range=["2020-01-01", "2020-01-02"], + var_lst=["rain", "inflow"], + time_unit="3h", + ) + + assert "3h" in result, "augmented read did not return the '3h' entry" + returned_basins = result["3h"]["basin"].values.tolist() + assert returned_basins == ["01015500"], ( + f"expected only ['01015500'], got {returned_basins}" + ) + + +# ── GREEN spec test: the write side uses the ``basin`` dimension ──────────── + + +def test_create_xarray_dataset_from_augdf_uses_basin_coord(floodevent_datasource): + """Write-end format uses the ``basin`` coord (never ``gage_id``). + + This documents the source of the mismatch: the cache files carry ``basin``, + so the read side must filter on ``basin`` too. + """ + n_time = 8 + df = pd.DataFrame( + { + "time": pd.date_range("2020-01-01", periods=n_time, freq="3h"), + "rain": np.ones(n_time), + "inflow": np.ones(n_time) * 10.0, + "flood_event": np.zeros(n_time, dtype=np.int32), + "ES": np.ones(n_time) * 0.5, + } + ) + + ds = floodevent_datasource.create_xarray_dataset_from_augdf( + df, station_id="01015500", time_unit="3h" + ) + + assert "basin" in ds.coords, "write side must expose a 'basin' coord" + assert "gage_id" not in ds.coords, "write side must NOT expose 'gage_id'" + assert ds["basin"].values.tolist() == ["01015500"] diff --git a/tests/test_fix_resolver_config.py b/tests/test_fix_resolver_config.py new file mode 100644 index 0000000..1ae2f04 --- /dev/null +++ b/tests/test_fix_resolver_config.py @@ -0,0 +1,78 @@ +"""RED regression tests for two resolver/config defects. + +These tests document the *desired* behavior and are expected to FAIL against +the current implementation (TDD RED phase / GAN discriminator). + +Defect A — `source` default bypasses `storage.default_source` + hydrodatasource.configs.data_resolver.resolve_data_path / open_dataset + hardcode ``source: Literal["local", "cloud"] = "local"``, so they always + forward an explicit ``"local"`` to the underlying hydrodataset resolver. + The underlying resolver's fallback to ``storage.default_source`` (when + ``source is None``) is therefore never reached. + +Defect B — non-empty settings without a ``storage`` key are silently ignored + hydrodatasource.configs.config._init_settings only warns when the loaded + settings dict is *empty*. A non-empty dict that lacks the ``storage`` + key silently falls back to the default local root with no UserWarning. +""" + +import pytest + +from hydrodatasource.configs import config as config_module +from hydrodatasource.configs.data_resolver import ( + ResolverContext, + resolve_data_path, +) + + +def _make_local_dataset_dir(tmp_path): + """Create ``{root}/projects/songliao/event`` under *tmp_path*. + + ``_resolve_local`` requires both ``storage.local.root`` and the resolved + registry-relative directory to exist on disk. + """ + dataset_dir = tmp_path / "projects" / "songliao" / "event" + dataset_dir.mkdir(parents=True) + return dataset_dir + + +class TestResolveDataPathDefaultSource: + """Defect A: default source must follow ``storage.default_source``.""" + + def test_default_cloud_returns_s3_uri(self, tmp_path): + """Omitted source + ``default_source: cloud`` must yield an s3:// URI.""" + ctx = ResolverContext( + storage={"default_source": "cloud", "s3": {"bucket": "b", "prefix": "p"}}, + project_root=tmp_path, + ) + + uri = resolve_data_path("songliao_event", ctx=ctx) + + assert uri.startswith("s3://") + + def test_default_local_returns_local_path(self, tmp_path): + """Omitted source + ``default_source: local`` must yield a local path.""" + dataset_dir = _make_local_dataset_dir(tmp_path) + ctx = ResolverContext( + storage={"default_source": "local", "local": {"root": str(tmp_path)}}, + project_root=tmp_path, + ) + + uri = resolve_data_path("songliao_event", ctx=ctx) + + assert not uri.startswith("s3://") + assert uri == str(dataset_dir) + + +class TestInitSettingsMissingStorageWarning: + """Defect B: non-empty settings lacking a ``storage`` key must warn.""" + + def test_non_empty_missing_storage_warns(self, monkeypatch): + """A UserWarning mentioning 'storage' is expected when settings lack it.""" + monkeypatch.setattr(config_module, "_initialized", False) + monkeypatch.setattr( + config_module, "_load_settings_from_file", lambda: {"foo": "bar"} + ) + + with pytest.warns(UserWarning, match="storage"): + config_module._init_settings() diff --git a/tests/test_hydroutils_compat.py b/tests/test_hydroutils_compat.py new file mode 100644 index 0000000..ed51243 --- /dev/null +++ b/tests/test_hydroutils_compat.py @@ -0,0 +1,110 @@ +""" +Regression tests for the hydroutils 0.1.0 -> 0.2.0 migration. + +Background +---------- +hydroutils 0.2.0 removed the `hydroutils/hydro_log.py` module (which provided +the `hydro_logger` class decorator and `HydroWarning`) and dropped the +`from .hydro_log import *` re-export in `hydroutils/__init__.py`. + +As a result, the following four modules currently fail at import time +because they still import from the removed `hydro_log` module or from the +removed re-exported `hydro_logger` name: + + - hydrodatasource.reader.gages + - hydrodatasource.reader.rainfall_reader + - hydrodatasource.cleaner.rainfall_cleaner + - hydrodatasource.cleaner.rsvr_inflow_cleaner + +The migration replaces every use of `hydro_log` with the Python standard +`logging` module and bumps the `hydroutils` dependency floor to `>=0.2.0`. +These tests pin that behavior. + +All imports of the affected modules happen *inside* the test functions so a +broken import surfaces as a per-test failure instead of a collection error. +""" + +import importlib +import logging +import tomllib +from pathlib import Path + +from packaging.requirements import Requirement + + +# --- 1. The four affected modules import cleanly ----------------------------- + +def test_gages_module_importable(): + assert importlib.import_module("hydrodatasource.reader.gages") is not None + + +def test_rainfall_reader_module_importable(): + assert ( + importlib.import_module("hydrodatasource.reader.rainfall_reader") is not None + ) + + +def test_rainfall_cleaner_module_importable(): + assert ( + importlib.import_module("hydrodatasource.cleaner.rainfall_cleaner") is not None + ) + + +def test_rsvr_inflow_cleaner_module_importable(): + assert ( + importlib.import_module("hydrodatasource.cleaner.rsvr_inflow_cleaner") + is not None + ) + + +# --- 2. Cleaner/reader classes expose a stdlib logging.Logger ---------------- + +def test_rainfall_cleaner_class_logger(): + from hydrodatasource.cleaner.rainfall_cleaner import RainfallCleaner + + assert isinstance(RainfallCleaner.logger, logging.Logger) + RainfallCleaner.logger.info("smoke") + + +def test_rainfall_analyzer_class_logger(): + from hydrodatasource.cleaner.rainfall_cleaner import RainfallAnalyzer + + assert isinstance(RainfallAnalyzer.logger, logging.Logger) + RainfallAnalyzer.logger.info("smoke") + + +def test_reservoir_inflow_backtrack_class_logger(): + from hydrodatasource.cleaner.rsvr_inflow_cleaner import ReservoirInflowBacktrack + + assert isinstance(ReservoirInflowBacktrack.logger, logging.Logger) + ReservoirInflowBacktrack.logger.info("smoke") + + +# --- 3. gages module-level logger -------------------------------------------- + +def test_gages_module_logger(): + import hydrodatasource.reader.gages as gages + + assert isinstance(gages.logger, logging.Logger) + gages.logger.info("smoke") + + +# --- 4. hydroutils dependency floor bumped to 0.2.0 -------------------------- + +def test_pyproject_requires_hydroutils_0_2_or_newer(): + toml_path = Path(__file__).resolve().parents[1] / "pyproject.toml" + with toml_path.open("rb") as f: + data = tomllib.load(f) + + deps = data["project"]["dependencies"] + hydroutils_deps = [ + d for d in deps if d.strip().lower().startswith("hydroutils") + ] + assert len(hydroutils_deps) == 1, ( + "expected exactly one hydroutils entry in [project].dependencies" + ) + + req = Requirement(hydroutils_deps[0]) + assert str(req.specifier) == ">=0.2.0", ( + f"hydroutils lower bound should be >=0.2.0, got {req.specifier!r}" + ) diff --git a/tests/test_package_exports.py b/tests/test_package_exports.py index 50877c8..e50806d 100644 --- a/tests/test_package_exports.py +++ b/tests/test_package_exports.py @@ -59,12 +59,23 @@ def test_hydrodatasource_aliases_total_count(self): ) def test_merged_total_count(self): - """Total merged aliases: 11 hydrodatasource + 26 hydrodataset = 37.""" + """Total merged aliases = hydrodatasource subset + hydrodataset subset (no overlap). + + 合并后的 READER_ALIASES 按 "category" 划分成两个互不相交的子集， + 因此总数等于两个子集计数之和。用并集恒等式代替硬编码总数， + 避免 hydrodataset 增减公共数据集时总数漂移导致测试失配。 + """ from hydrodatasource import READER_ALIASES - assert len(READER_ALIASES) == 37, ( - f"Expected 37 total aliases (11 HDS + 26 HD), got {len(READER_ALIASES)}" - ) + hds = { + k: v for k, v in READER_ALIASES.items() if v.get("category") == "hydrodatasource" + } + hd = {k: v for k, v in READER_ALIASES.items() if v.get("category") == "hydrodataset"} + + # 并集恒等式：每个别名恰好属于一个类别 + assert len(READER_ALIASES) == len(hds) + len(hd) + # 防倒退下限：别名总数不得少于上一版本 + assert len(READER_ALIASES) >= 37 def test_resolver_context_importable_from_package(self): """ResolverContext is importable from hydrodatasource.""" diff --git a/tests/test_tg_hydro_datasource.py b/tests/test_tg_hydro_datasource.py index 8dbf50d..e5fcc40 100644 --- a/tests/test_tg_hydro_datasource.py +++ b/tests/test_tg_hydro_datasource.py @@ -12,8 +12,8 @@ import xarray as xr import geopandas as gpd from shapely.geometry import Polygon -import hydrodatasource.configs.config as conf +from hydrodatasource.reader import data_source from hydrodatasource.reader.data_source import TgHydroDatasource @@ -65,8 +65,10 @@ def _create_lstm_predictions(base): lstm_dataset.to_netcdf(lstm_pred_file) # 同步写入到缓存目录，满足 TgHydroDatasource 的路径约定 - os.makedirs(conf.CACHE_DIR, exist_ok=True) - lstm_dataset.to_netcdf(os.path.join(conf.CACHE_DIR, "lstmpred.nc")) + # 注意：data_source.py 顶部为模块级绑定 CACHE_DIR，必须通过 data_source.CACHE_DIR 访问， + # 这样 fixture 里 monkeypatch 的临时缓存目录才会生效，避免污染用户 home 下的真实缓存。 + os.makedirs(data_source.CACHE_DIR, exist_ok=True) + lstm_dataset.to_netcdf(os.path.join(data_source.CACHE_DIR, "lstmpred.nc")) return lstm_pred_file @@ -82,6 +84,9 @@ def _create_timeseries_csv(base): "station_6", ] np.random.seed(123) + base_ts_dir = os.path.join(base, "timeseries", "1D") + inter_ts_dir = os.path.join(base, "intermediate", "timeseries", "1D") + os.makedirs(inter_ts_dir, exist_ok=True) for name in basin_names: df = pd.DataFrame( { @@ -90,7 +95,9 @@ def _create_timeseries_csv(base): "streamflow": np.random.rand(len(time_range)) * 50, } ) - df.to_csv(os.path.join(base, "timeseries", "1D", f"{name}.csv"), index=False) + df.to_csv(os.path.join(base_ts_dir, f"{name}.csv"), index=False) + # 同步写入 intermediate 目录，使 combine=True 路径能读取到 intermediate 数据 + df.to_csv(os.path.join(inter_ts_dir, f"{name}.csv"), index=False) def _create_shapes(base): @@ -155,7 +162,13 @@ def _create_attributes(base): @pytest.fixture -def tg_dataset(tmp_path): +def tg_dataset(tmp_path, monkeypatch): + # 隔离缓存目录：data_source.py 在模块级绑定 CACHE_DIR，这里 monkeypatch 该模块属性， + # 使缓存写入（含 lstmpred.nc）落在 tmp_path 下，避免污染用户 home 下的真实 CACHE_DIR。 + cache_dir = tmp_path / "cache" + monkeypatch.setattr("hydrodatasource.reader.data_source.CACHE_DIR", str(cache_dir)) + os.makedirs(cache_dir, exist_ok=True) + dataset_name = "test_tg_dataset" base = _create_dirs(tmp_path, dataset_name) _create_graph_dict(base) @@ -278,17 +291,22 @@ def test_read_ts_xrdataset_combine_outputs(tg_dataset): print(f"[TS-COMBINE] var={v} sample (first 6 basins x first 3 time):\n", sample.to_pandas()) # lstm_pred 仅注入到 base 部分；intermediate 部分应为空或 NaN + # 注意：combine 后 base 与 intermediate 的 basin 名相同，basin 坐标存在重复， + # 不能用 .sel(basin=...) 标签索引，需按 source 掩码做位置索引。 assert "lstm_pred" in ds.data_vars assert set(ds["lstm_pred"].dims) == {"basin", "time"} inter_mask = ds["source"].values == "intermediate" inter_basins = ds["basin"].values[inter_mask] base_mask = ds["source"].values == "base" - base_basins = ds["basin"].values[base_mask] # 打印样本 - lstm_base_sample = ds["lstm_pred"].transpose("basin", "time").sel(basin=base_basins).isel(time=slice(0, 3)) + lstm_base_sample = ds["lstm_pred"].transpose("basin", "time").isel( + basin=np.where(base_mask)[0], time=slice(0, 3) + ) print("[TS-COMBINE] var=lstm_pred base sample:\n", lstm_base_sample.to_pandas().head()) if len(inter_basins) > 0: - lstm_inter_sample = ds["lstm_pred"].transpose("basin", "time").sel(basin=inter_basins).isel(time=slice(0, 3)) + lstm_inter_sample = ds["lstm_pred"].transpose("basin", "time").isel( + basin=np.where(inter_mask)[0], time=slice(0, 3) + ) print("[TS-COMBINE] var=lstm_pred intermediate sample:\n", lstm_inter_sample.to_pandas().head()) # 检查 intermediate 部分为 NaN assert np.isnan(lstm_inter_sample.to_numpy()).all() diff --git a/uv.lock b/uv.lock index bb9a48f..d16c669 100644 --- a/uv.lock +++ b/uv.lock @@ -247,6 +247,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl", hash = "sha256:fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b", size = 13929, upload-time = "2024-07-26T18:15:02.05Z" }, ] +[[package]] +name = "annotated-doc" +version = "0.0.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/8e/38aa427ed5402449e226975b649c5dc73ccadfefeb95e6aecb8f8ea4b6b6/annotated_doc-0.0.5.tar.gz", hash = "sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb", size = 10758, upload-time = "2026-07-28T13:50:58.129Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl", hash = "sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101", size = 5302, upload-time = "2026-07-28T13:50:57.239Z" }, +] + [[package]] name = "appnope" version = "0.1.4" @@ -277,54 +286,6 @@ all = [ { name = "xarray" }, ] -[[package]] -name = "argon2-cffi" -version = "25.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "argon2-cffi-bindings" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0e/89/ce5af8a7d472a67cc819d5d998aa8c82c5d860608c4db9f46f1162d7dab9/argon2_cffi-25.1.0.tar.gz", hash = "sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1", size = 45706, upload-time = "2025-06-03T06:55:32.073Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl", hash = "sha256:fdc8b074db390fccb6eb4a3604ae7231f219aa669a2652e0f20e16ba513d5741", size = 14657, upload-time = "2025-06-03T06:55:30.804Z" }, -] - -[[package]] -name = "argon2-cffi-bindings" -version = "25.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5c/2d/db8af0df73c1cf454f71b2bbe5e356b8c1f8041c979f505b3d3186e520a9/argon2_cffi_bindings-25.1.0.tar.gz", hash = "sha256:b957f3e6ea4d55d820e40ff76f450952807013d361a65d7f28acc0acbf29229d", size = 1783441, upload-time = "2025-07-30T10:02:05.147Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/60/97/3c0a35f46e52108d4707c44b95cfe2afcafc50800b5450c197454569b776/argon2_cffi_bindings-25.1.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:3d3f05610594151994ca9ccb3c771115bdb4daef161976a266f0dd8aa9996b8f", size = 54393, upload-time = "2025-07-30T10:01:40.97Z" }, - { url = "https://files.pythonhosted.org/packages/9d/f4/98bbd6ee89febd4f212696f13c03ca302b8552e7dbf9c8efa11ea4a388c3/argon2_cffi_bindings-25.1.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8b8efee945193e667a396cbc7b4fb7d357297d6234d30a489905d96caabde56b", size = 29328, upload-time = "2025-07-30T10:01:41.916Z" }, - { url = "https://files.pythonhosted.org/packages/43/24/90a01c0ef12ac91a6be05969f29944643bc1e5e461155ae6559befa8f00b/argon2_cffi_bindings-25.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3c6702abc36bf3ccba3f802b799505def420a1b7039862014a65db3205967f5a", size = 31269, upload-time = "2025-07-30T10:01:42.716Z" }, - { url = "https://files.pythonhosted.org/packages/d4/d3/942aa10782b2697eee7af5e12eeff5ebb325ccfb86dd8abda54174e377e4/argon2_cffi_bindings-25.1.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a1c70058c6ab1e352304ac7e3b52554daadacd8d453c1752e547c76e9c99ac44", size = 86558, upload-time = "2025-07-30T10:01:43.943Z" }, - { url = "https://files.pythonhosted.org/packages/0d/82/b484f702fec5536e71836fc2dbc8c5267b3f6e78d2d539b4eaa6f0db8bf8/argon2_cffi_bindings-25.1.0-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2fd3bfbff3c5d74fef31a722f729bf93500910db650c925c2d6ef879a7e51cb", size = 92364, upload-time = "2025-07-30T10:01:44.887Z" }, - { url = "https://files.pythonhosted.org/packages/c9/c1/a606ff83b3f1735f3759ad0f2cd9e038a0ad11a3de3b6c673aa41c24bb7b/argon2_cffi_bindings-25.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c4f9665de60b1b0e99bcd6be4f17d90339698ce954cfd8d9cf4f91c995165a92", size = 85637, upload-time = "2025-07-30T10:01:46.225Z" }, - { url = "https://files.pythonhosted.org/packages/44/b4/678503f12aceb0262f84fa201f6027ed77d71c5019ae03b399b97caa2f19/argon2_cffi_bindings-25.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ba92837e4a9aa6a508c8d2d7883ed5a8f6c308c89a4790e1e447a220deb79a85", size = 91934, upload-time = "2025-07-30T10:01:47.203Z" }, - { url = "https://files.pythonhosted.org/packages/f0/c7/f36bd08ef9bd9f0a9cff9428406651f5937ce27b6c5b07b92d41f91ae541/argon2_cffi_bindings-25.1.0-cp314-cp314t-win32.whl", hash = "sha256:84a461d4d84ae1295871329b346a97f68eade8c53b6ed9a7ca2d7467f3c8ff6f", size = 28158, upload-time = "2025-07-30T10:01:48.341Z" }, - { url = "https://files.pythonhosted.org/packages/b3/80/0106a7448abb24a2c467bf7d527fe5413b7fdfa4ad6d6a96a43a62ef3988/argon2_cffi_bindings-25.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b55aec3565b65f56455eebc9b9f34130440404f27fe21c3b375bf1ea4d8fbae6", size = 32597, upload-time = "2025-07-30T10:01:49.112Z" }, - { url = "https://files.pythonhosted.org/packages/05/b8/d663c9caea07e9180b2cb662772865230715cbd573ba3b5e81793d580316/argon2_cffi_bindings-25.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:87c33a52407e4c41f3b70a9c2d3f6056d88b10dad7695be708c5021673f55623", size = 28231, upload-time = "2025-07-30T10:01:49.92Z" }, - { url = "https://files.pythonhosted.org/packages/1d/57/96b8b9f93166147826da5f90376e784a10582dd39a393c99bb62cfcf52f0/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:aecba1723ae35330a008418a91ea6cfcedf6d31e5fbaa056a166462ff066d500", size = 54121, upload-time = "2025-07-30T10:01:50.815Z" }, - { url = "https://files.pythonhosted.org/packages/0a/08/a9bebdb2e0e602dde230bdde8021b29f71f7841bd54801bcfd514acb5dcf/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:2630b6240b495dfab90aebe159ff784d08ea999aa4b0d17efa734055a07d2f44", size = 29177, upload-time = "2025-07-30T10:01:51.681Z" }, - { url = "https://files.pythonhosted.org/packages/b6/02/d297943bcacf05e4f2a94ab6f462831dc20158614e5d067c35d4e63b9acb/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:7aef0c91e2c0fbca6fc68e7555aa60ef7008a739cbe045541e438373bc54d2b0", size = 31090, upload-time = "2025-07-30T10:01:53.184Z" }, - { url = "https://files.pythonhosted.org/packages/c1/93/44365f3d75053e53893ec6d733e4a5e3147502663554b4d864587c7828a7/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e021e87faa76ae0d413b619fe2b65ab9a037f24c60a1e6cc43457ae20de6dc6", size = 81246, upload-time = "2025-07-30T10:01:54.145Z" }, - { url = "https://files.pythonhosted.org/packages/09/52/94108adfdd6e2ddf58be64f959a0b9c7d4ef2fa71086c38356d22dc501ea/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d3e924cfc503018a714f94a49a149fdc0b644eaead5d1f089330399134fa028a", size = 87126, upload-time = "2025-07-30T10:01:55.074Z" }, - { url = "https://files.pythonhosted.org/packages/72/70/7a2993a12b0ffa2a9271259b79cc616e2389ed1a4d93842fac5a1f923ffd/argon2_cffi_bindings-25.1.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c87b72589133f0346a1cb8d5ecca4b933e3c9b64656c9d175270a000e73b288d", size = 80343, upload-time = "2025-07-30T10:01:56.007Z" }, - { url = "https://files.pythonhosted.org/packages/78/9a/4e5157d893ffc712b74dbd868c7f62365618266982b64accab26bab01edc/argon2_cffi_bindings-25.1.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1db89609c06afa1a214a69a462ea741cf735b29a57530478c06eb81dd403de99", size = 86777, upload-time = "2025-07-30T10:01:56.943Z" }, - { url = "https://files.pythonhosted.org/packages/74/cd/15777dfde1c29d96de7f18edf4cc94c385646852e7c7b0320aa91ccca583/argon2_cffi_bindings-25.1.0-cp39-abi3-win32.whl", hash = "sha256:473bcb5f82924b1becbb637b63303ec8d10e84c8d241119419897a26116515d2", size = 27180, upload-time = "2025-07-30T10:01:57.759Z" }, - { url = "https://files.pythonhosted.org/packages/e2/c6/a759ece8f1829d1f162261226fbfd2c6832b3ff7657384045286d2afa384/argon2_cffi_bindings-25.1.0-cp39-abi3-win_amd64.whl", hash = "sha256:a98cd7d17e9f7ce244c0803cad3c23a7d379c301ba618a5fa76a67d116618b98", size = 31715, upload-time = "2025-07-30T10:01:58.56Z" }, - { url = "https://files.pythonhosted.org/packages/42/b9/f8d6fa329ab25128b7e98fd83a3cb34d9db5b059a9847eddb840a0af45dd/argon2_cffi_bindings-25.1.0-cp39-abi3-win_arm64.whl", hash = "sha256:b0fdbcf513833809c882823f98dc2f931cf659d9a1429616ac3adebb49f5db94", size = 27149, upload-time = "2025-07-30T10:01:59.329Z" }, - { url = "https://files.pythonhosted.org/packages/11/2d/ba4e4ca8d149f8dcc0d952ac0967089e1d759c7e5fcf0865a317eb680fbb/argon2_cffi_bindings-25.1.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:6dca33a9859abf613e22733131fc9194091c1fa7cb3e131c143056b4856aa47e", size = 24549, upload-time = "2025-07-30T10:02:00.101Z" }, - { url = "https://files.pythonhosted.org/packages/5c/82/9b2386cc75ac0bd3210e12a44bfc7fd1632065ed8b80d573036eecb10442/argon2_cffi_bindings-25.1.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:21378b40e1b8d1655dd5310c84a40fc19a9aa5e6366e835ceb8576bf0fea716d", size = 25539, upload-time = "2025-07-30T10:02:00.929Z" }, - { url = "https://files.pythonhosted.org/packages/31/db/740de99a37aa727623730c90d92c22c9e12585b3c98c54b7960f7810289f/argon2_cffi_bindings-25.1.0-pp310-pypy310_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5d588dec224e2a83edbdc785a5e6f3c6cd736f46bfd4b441bbb5aa1f5085e584", size = 28467, upload-time = "2025-07-30T10:02:02.08Z" }, - { url = "https://files.pythonhosted.org/packages/71/7a/47c4509ea18d755f44e2b92b7178914f0c113946d11e16e626df8eaa2b0b/argon2_cffi_bindings-25.1.0-pp310-pypy310_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5acb4e41090d53f17ca1110c3427f0a130f944b896fc8c83973219c97f57b690", size = 27355, upload-time = "2025-07-30T10:02:02.867Z" }, - { url = "https://files.pythonhosted.org/packages/ee/82/82745642d3c46e7cea25e1885b014b033f4693346ce46b7f47483cf5d448/argon2_cffi_bindings-25.1.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:da0c79c23a63723aa5d782250fbf51b768abca630285262fb5144ba5ae01e520", size = 29187, upload-time = "2025-07-30T10:02:03.674Z" }, -] - [[package]] name = "asciitree" version = "0.3.3" @@ -515,20 +476,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/96/9a/8e641b5415e12036d8a206147b8229d917a767b7d939521458d90feddcf5/bokeh-3.8.0-py3-none-any.whl", hash = "sha256:117c5e559231ad39fef87891a1a1b62b3bfefbaa47d536023537338f46015841", size = 7205343, upload-time = "2025-08-29T12:16:52.77Z" }, ] -[[package]] -name = "boto3" -version = "1.40.61" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "botocore" }, - { name = "jmespath" }, - { name = "s3transfer" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ed/f9/6ef8feb52c3cce5ec3967a535a6114b57ac7949fd166b0f3090c2b06e4e5/boto3-1.40.61.tar.gz", hash = "sha256:d6c56277251adf6c2bdd25249feae625abe4966831676689ff23b4694dea5b12", size = 111535, upload-time = "2025-10-28T19:26:57.247Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/61/24/3bf865b07d15fea85b63504856e137029b6acbc73762496064219cdb265d/boto3-1.40.61-py3-none-any.whl", hash = "sha256:6b9c57b2a922b5d8c17766e29ed792586a818098efe84def27c8f582b33f898c", size = 139321, upload-time = "2025-10-28T19:26:55.007Z" }, -] - [[package]] name = "botocore" version = "1.40.61" @@ -2382,8 +2329,10 @@ dependencies = [ { name = "pint-xarray", version = "0.6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "py7zr" }, { name = "requests" }, + { name = "s3fs" }, { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "scipy", version = "1.16.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "typer" }, { name = "tzfpy" }, { name = "xarray" }, { name = "xlrd" }, @@ -2409,7 +2358,7 @@ requires-dist = [ { name = "grip", marker = "extra == 'docs'" }, { name = "hatch-vcs", marker = "extra == 'release'" }, { name = "hatchling", marker = "extra == 'release'" }, - { name = "hydroutils" }, + { name = "hydroutils", specifier = ">=0.2.0" }, { name = "ipykernel", marker = "extra == 'docs'" }, { name = "mkdocs-git-revision-date-plugin", marker = "extra == 'docs'" }, { name = "mkdocs-jupyter", marker = "extra == 'docs'" }, @@ -2425,12 +2374,14 @@ requires-dist = [ { name = "pytest", marker = "extra == 'dev'", specifier = ">=3" }, { name = "pytest-runner", marker = "extra == 'dev'" }, { name = "requests", specifier = ">=2.32.4" }, + { name = "s3fs", specifier = ">=2024.12" }, { name = "scipy" }, { name = "sphinx", marker = "extra == 'dev'" }, { name = "sphinx", marker = "extra == 'docs'" }, { name = "tox", marker = "extra == 'dev'" }, { name = "twine", marker = "extra == 'dev'" }, { name = "twine", marker = "extra == 'release'" }, + { name = "typer", specifier = ">=0.26.8" }, { name = "tzfpy" }, { name = "watchdog", marker = "extra == 'dev'" }, { name = "wheel", marker = "extra == 'dev'" }, @@ -2530,7 +2481,7 @@ requires-dist = [ { name = "h5py" }, { name = "hydrodataset", editable = "../hydrodataset" }, { name = "hydrotopo" }, - { name = "hydroutils", specifier = ">=0.1.0" }, + { name = "hydroutils", specifier = ">=0.2.0" }, { name = "intake" }, { name = "ipykernel", marker = "extra == 'docs'" }, { name = "kerchunk" }, @@ -2628,15 +2579,13 @@ wheels = [ [[package]] name = "hydroutils" -version = "0.1.0" +version = "0.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "async-retriever" }, - { name = "boto3" }, { name = "cartopy" }, { name = "hydroerr" }, { name = "matplotlib" }, - { name = "minio" }, { name = "mkdocs-git-revision-date-localized-plugin" }, { name = "mkdocs-git-revision-date-plugin" }, { name = "mkdocstrings-python" }, @@ -2648,17 +2597,19 @@ dependencies = [ { name = "pint", version = "0.25.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "pint-xarray", version = "0.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "pint-xarray", version = "0.6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "pytz" }, { name = "requests" }, { name = "rich" }, - { name = "s3fs" }, + { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "scipy", version = "1.16.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "seaborn" }, { name = "tqdm" }, { name = "tzfpy" }, { name = "xarray" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ce/ce/4ef3af2f2bd5afcd4ba35bccd761f7aa8a8f44e7acf6f173849955ac544e/hydroutils-0.1.0.tar.gz", hash = "sha256:17272415c5b6020846f0cf526a56eb26f74a8ac2d501cc97543f3832aeccefa5", size = 94104, upload-time = "2025-10-28T01:24:15.225Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f0/e0/5ab8f3d8a62223b3fbf3a32b4a75a6519eda92eaf75c2277ad3b86ac2007/hydroutils-0.2.0.tar.gz", hash = "sha256:d77e0624b54110dafbb5132279b6d1650374e6fe80e7ec55b0409221ece962e0", size = 92070, upload-time = "2026-08-15T02:47:44.647Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c6/c7/b4ac0c61e452c263e10b495c8f7dfa812a878622bc58b15a7dc2606942b1/hydroutils-0.1.0-py3-none-any.whl", hash = "sha256:be160af5a85d96d9dea85ce0ccf7a019ad269c109228779e9aec0aa4295cf722", size = 65333, upload-time = "2025-10-28T01:24:13.525Z" }, + { url = "https://files.pythonhosted.org/packages/fc/15/e5f269e03dd3dacbdcf508f1b03bb4f3947d6e12adba2ba60a0364fb6d09/hydroutils-0.2.0-py3-none-any.whl", hash = "sha256:a0d9c2c24ec51f42871e36d7ecdc63bdb41372aa02bae7b24e7d7f683fe48969", size = 64167, upload-time = "2026-08-15T02:47:43.1Z" }, ] [[package]] @@ -3704,22 +3655,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/19/04f9b178c2d8a15b076c8b5140708fa6ffc5601fb6f1e975537072df5b2a/mergedeep-1.3.4-py3-none-any.whl", hash = "sha256:70775750742b25c0d8f36c55aed03d24c3384d17c951b3175d898bd778ef0307", size = 6354, upload-time = "2021-02-05T18:55:29.583Z" }, ] -[[package]] -name = "minio" -version = "7.2.18" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "argon2-cffi" }, - { name = "certifi" }, - { name = "pycryptodome" }, - { name = "typing-extensions" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/71/99/1ad8733fa3f2fa82726e470f8c321e17f9321083b234ab45ad6b59d80d9f/minio-7.2.18.tar.gz", hash = "sha256:173402a5716099159c5659f9de75be204ebe248557b9f1cc9cf45aa70e9d3024", size = 135544, upload-time = "2025-09-29T17:00:28.238Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7d/ae/f32695da4f93de50dd7075100dab8cf689a9d96270f58ce6f940fd044a3e/minio-7.2.18-py3-none-any.whl", hash = "sha256:f23a6edbff8d0bc4b5c1a61b2628a01c5a3342aefc613ff9c276012e6321108f", size = 93120, upload-time = "2025-09-29T17:00:26.86Z" }, -] - [[package]] name = "mistune" version = "3.1.4" @@ -5514,41 +5449,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/e3/59cd50310fc9b59512193629e1984c1f95e5c8ae6e5d8c69532ccc65a7fe/pycparser-2.23-py3-none-any.whl", hash = "sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934", size = 118140, upload-time = "2025-09-09T13:23:46.651Z" }, ] -[[package]] -name = "pycryptodome" -version = "3.23.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8e/a6/8452177684d5e906854776276ddd34eca30d1b1e15aa1ee9cefc289a33f5/pycryptodome-3.23.0.tar.gz", hash = "sha256:447700a657182d60338bab09fdb27518f8856aecd80ae4c6bdddb67ff5da44ef", size = 4921276, upload-time = "2025-05-17T17:21:45.242Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/04/5d/bdb09489b63cd34a976cc9e2a8d938114f7a53a74d3dd4f125ffa49dce82/pycryptodome-3.23.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:0011f7f00cdb74879142011f95133274741778abba114ceca229adbf8e62c3e4", size = 2495152, upload-time = "2025-05-17T17:20:20.833Z" }, - { url = "https://files.pythonhosted.org/packages/a7/ce/7840250ed4cc0039c433cd41715536f926d6e86ce84e904068eb3244b6a6/pycryptodome-3.23.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:90460fc9e088ce095f9ee8356722d4f10f86e5be06e2354230a9880b9c549aae", size = 1639348, upload-time = "2025-05-17T17:20:23.171Z" }, - { url = "https://files.pythonhosted.org/packages/ee/f0/991da24c55c1f688d6a3b5a11940567353f74590734ee4a64294834ae472/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4764e64b269fc83b00f682c47443c2e6e85b18273712b98aa43bcb77f8570477", size = 2184033, upload-time = "2025-05-17T17:20:25.424Z" }, - { url = "https://files.pythonhosted.org/packages/54/16/0e11882deddf00f68b68dd4e8e442ddc30641f31afeb2bc25588124ac8de/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb8f24adb74984aa0e5d07a2368ad95276cf38051fe2dc6605cbcf482e04f2a7", size = 2270142, upload-time = "2025-05-17T17:20:27.808Z" }, - { url = "https://files.pythonhosted.org/packages/d5/fc/4347fea23a3f95ffb931f383ff28b3f7b1fe868739182cb76718c0da86a1/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d97618c9c6684a97ef7637ba43bdf6663a2e2e77efe0f863cce97a76af396446", size = 2309384, upload-time = "2025-05-17T17:20:30.765Z" }, - { url = "https://files.pythonhosted.org/packages/6e/d9/c5261780b69ce66d8cfab25d2797bd6e82ba0241804694cd48be41add5eb/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9a53a4fe5cb075075d515797d6ce2f56772ea7e6a1e5e4b96cf78a14bac3d265", size = 2183237, upload-time = "2025-05-17T17:20:33.736Z" }, - { url = "https://files.pythonhosted.org/packages/5a/6f/3af2ffedd5cfa08c631f89452c6648c4d779e7772dfc388c77c920ca6bbf/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:763d1d74f56f031788e5d307029caef067febf890cd1f8bf61183ae142f1a77b", size = 2343898, upload-time = "2025-05-17T17:20:36.086Z" }, - { url = "https://files.pythonhosted.org/packages/9a/dc/9060d807039ee5de6e2f260f72f3d70ac213993a804f5e67e0a73a56dd2f/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:954af0e2bd7cea83ce72243b14e4fb518b18f0c1649b576d114973e2073b273d", size = 2269197, upload-time = "2025-05-17T17:20:38.414Z" }, - { url = "https://files.pythonhosted.org/packages/f9/34/e6c8ca177cb29dcc4967fef73f5de445912f93bd0343c9c33c8e5bf8cde8/pycryptodome-3.23.0-cp313-cp313t-win32.whl", hash = "sha256:257bb3572c63ad8ba40b89f6fc9d63a2a628e9f9708d31ee26560925ebe0210a", size = 1768600, upload-time = "2025-05-17T17:20:40.688Z" }, - { url = "https://files.pythonhosted.org/packages/e4/1d/89756b8d7ff623ad0160f4539da571d1f594d21ee6d68be130a6eccb39a4/pycryptodome-3.23.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6501790c5b62a29fcb227bd6b62012181d886a767ce9ed03b303d1f22eb5c625", size = 1799740, upload-time = "2025-05-17T17:20:42.413Z" }, - { url = "https://files.pythonhosted.org/packages/5d/61/35a64f0feaea9fd07f0d91209e7be91726eb48c0f1bfc6720647194071e4/pycryptodome-3.23.0-cp313-cp313t-win_arm64.whl", hash = "sha256:9a77627a330ab23ca43b48b130e202582e91cc69619947840ea4d2d1be21eb39", size = 1703685, upload-time = "2025-05-17T17:20:44.388Z" }, - { url = "https://files.pythonhosted.org/packages/db/6c/a1f71542c969912bb0e106f64f60a56cc1f0fabecf9396f45accbe63fa68/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:187058ab80b3281b1de11c2e6842a357a1f71b42cb1e15bce373f3d238135c27", size = 2495627, upload-time = "2025-05-17T17:20:47.139Z" }, - { url = "https://files.pythonhosted.org/packages/6e/4e/a066527e079fc5002390c8acdd3aca431e6ea0a50ffd7201551175b47323/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:cfb5cd445280c5b0a4e6187a7ce8de5a07b5f3f897f235caa11f1f435f182843", size = 1640362, upload-time = "2025-05-17T17:20:50.392Z" }, - { url = "https://files.pythonhosted.org/packages/50/52/adaf4c8c100a8c49d2bd058e5b551f73dfd8cb89eb4911e25a0c469b6b4e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67bd81fcbe34f43ad9422ee8fd4843c8e7198dd88dd3d40e6de42ee65fbe1490", size = 2182625, upload-time = "2025-05-17T17:20:52.866Z" }, - { url = "https://files.pythonhosted.org/packages/5f/e9/a09476d436d0ff1402ac3867d933c61805ec2326c6ea557aeeac3825604e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8987bd3307a39bc03df5c8e0e3d8be0c4c3518b7f044b0f4c15d1aa78f52575", size = 2268954, upload-time = "2025-05-17T17:20:55.027Z" }, - { url = "https://files.pythonhosted.org/packages/f9/c5/ffe6474e0c551d54cab931918127c46d70cab8f114e0c2b5a3c071c2f484/pycryptodome-3.23.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa0698f65e5b570426fc31b8162ed4603b0c2841cbb9088e2b01641e3065915b", size = 2308534, upload-time = "2025-05-17T17:20:57.279Z" }, - { url = "https://files.pythonhosted.org/packages/18/28/e199677fc15ecf43010f2463fde4c1a53015d1fe95fb03bca2890836603a/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:53ecbafc2b55353edcebd64bf5da94a2a2cdf5090a6915bcca6eca6cc452585a", size = 2181853, upload-time = "2025-05-17T17:20:59.322Z" }, - { url = "https://files.pythonhosted.org/packages/ce/ea/4fdb09f2165ce1365c9eaefef36625583371ee514db58dc9b65d3a255c4c/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_i686.whl", hash = "sha256:156df9667ad9f2ad26255926524e1c136d6664b741547deb0a86a9acf5ea631f", size = 2342465, upload-time = "2025-05-17T17:21:03.83Z" }, - { url = "https://files.pythonhosted.org/packages/22/82/6edc3fc42fe9284aead511394bac167693fb2b0e0395b28b8bedaa07ef04/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:dea827b4d55ee390dc89b2afe5927d4308a8b538ae91d9c6f7a5090f397af1aa", size = 2267414, upload-time = "2025-05-17T17:21:06.72Z" }, - { url = "https://files.pythonhosted.org/packages/59/fe/aae679b64363eb78326c7fdc9d06ec3de18bac68be4b612fc1fe8902693c/pycryptodome-3.23.0-cp37-abi3-win32.whl", hash = "sha256:507dbead45474b62b2bbe318eb1c4c8ee641077532067fec9c1aa82c31f84886", size = 1768484, upload-time = "2025-05-17T17:21:08.535Z" }, - { url = "https://files.pythonhosted.org/packages/54/2f/e97a1b8294db0daaa87012c24a7bb714147c7ade7656973fd6c736b484ff/pycryptodome-3.23.0-cp37-abi3-win_amd64.whl", hash = "sha256:c75b52aacc6c0c260f204cbdd834f76edc9fb0d8e0da9fbf8352ef58202564e2", size = 1799636, upload-time = "2025-05-17T17:21:10.393Z" }, - { url = "https://files.pythonhosted.org/packages/18/3d/f9441a0d798bf2b1e645adc3265e55706aead1255ccdad3856dbdcffec14/pycryptodome-3.23.0-cp37-abi3-win_arm64.whl", hash = "sha256:11eeeb6917903876f134b56ba11abe95c0b0fd5e3330def218083c7d98bbcb3c", size = 1703675, upload-time = "2025-05-17T17:21:13.146Z" }, - { url = "https://files.pythonhosted.org/packages/d9/12/e33935a0709c07de084d7d58d330ec3f4daf7910a18e77937affdb728452/pycryptodome-3.23.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:ddb95b49df036ddd264a0ad246d1be5b672000f12d6961ea2c267083a5e19379", size = 1623886, upload-time = "2025-05-17T17:21:20.614Z" }, - { url = "https://files.pythonhosted.org/packages/22/0b/aa8f9419f25870889bebf0b26b223c6986652bdf071f000623df11212c90/pycryptodome-3.23.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e95564beb8782abfd9e431c974e14563a794a4944c29d6d3b7b5ea042110b4", size = 1672151, upload-time = "2025-05-17T17:21:22.666Z" }, - { url = "https://files.pythonhosted.org/packages/d4/5e/63f5cbde2342b7f70a39e591dbe75d9809d6338ce0b07c10406f1a140cdc/pycryptodome-3.23.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14e15c081e912c4b0d75632acd8382dfce45b258667aa3c67caf7a4d4c13f630", size = 1664461, upload-time = "2025-05-17T17:21:25.225Z" }, - { url = "https://files.pythonhosted.org/packages/d6/92/608fbdad566ebe499297a86aae5f2a5263818ceeecd16733006f1600403c/pycryptodome-3.23.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7fc76bf273353dc7e5207d172b83f569540fc9a28d63171061c42e361d22353", size = 1702440, upload-time = "2025-05-17T17:21:27.991Z" }, - { url = "https://files.pythonhosted.org/packages/d1/92/2eadd1341abd2989cce2e2740b4423608ee2014acb8110438244ee97d7ff/pycryptodome-3.23.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:45c69ad715ca1a94f778215a11e66b7ff989d792a4d63b68dc586a1da1392ff5", size = 1803005, upload-time = "2025-05-17T17:21:31.37Z" }, -] - [[package]] name = "pycryptodomex" version = "3.23.0" @@ -6782,18 +6682,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2d/fc/56cba14af8ad8fd020c85b6e44328520ac55939bb1f9d01444ad470504cb/s3fs-2025.10.0-py3-none-any.whl", hash = "sha256:da7ef25efc1541f5fca8e1116361e49ea1081f83f4e8001fbd77347c625da28a", size = 30357, upload-time = "2025-10-30T15:06:03.48Z" }, ] -[[package]] -name = "s3transfer" -version = "0.14.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "botocore" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/62/74/8d69dcb7a9efe8baa2046891735e5dfe433ad558ae23d9e3c14c633d1d58/s3transfer-0.14.0.tar.gz", hash = "sha256:eff12264e7c8b4985074ccce27a3b38a485bb7f7422cc8046fee9be4983e4125", size = 151547, upload-time = "2025-09-09T19:23:31.089Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/48/f0/ae7ca09223a81a1d890b2557186ea015f6e0502e9b8cb8e1813f1d8cfa4e/s3transfer-0.14.0-py3-none-any.whl", hash = "sha256:ea3b790c7077558ed1f02a3072fb3cb992bbbd253392f4b6e9e8976941c7d456", size = 85712, upload-time = "2025-09-09T19:23:30.041Z" }, -] - [[package]] name = "scikit-base" version = "0.12.6" @@ -7043,6 +6931,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9a/f6/f09272a71976dfc138129b8faf435d064a811ae2f708cb147dccdf7aacdb/shapely-2.1.2-cp314-cp314t-win_amd64.whl", hash = "sha256:0036ac886e0923417932c2e6369b6c52e38e0ff5d9120b90eef5cd9a5fc5cae9", size = 1796682, upload-time = "2025-09-24T13:51:39.233Z" }, ] +[[package]] +name = "shellingham" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, +] + [[package]] name = "six" version = "1.17.0" @@ -7456,6 +7353,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3a/7a/882d99539b19b1490cac5d77c67338d126e4122c8276bf640e411650c830/twine-6.2.0-py3-none-any.whl", hash = "sha256:418ebf08ccda9a8caaebe414433b0ba5e25eb5e4a927667122fbe8f829f985d8", size = 42727, upload-time = "2025-09-04T15:43:15.994Z" }, ] +[[package]] +name = "typer" +version = "0.27.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "rich" }, + { name = "shellingham" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ae/40/4a3db7990d1f62a53182aa96eaef57aeb2886a27f90a195bc66713565d31/typer-0.27.1.tar.gz", hash = "sha256:a79bef8469a79c45498e7b814ecf8d603cc7644e9acbd9e19cac0334240b18df", size = 203994, upload-time = "2026-08-03T14:41:03.438Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/89/9518bc0c3929bee36b3a4a8e3daddd6e03f92f9961c66d4983b837160543/typer-0.27.1-py3-none-any.whl", hash = "sha256:53150287edd11baeb4e4722c8e394fcdf8181c0ae89485cba8d25c778d5edd56", size = 122874, upload-time = "2026-08-03T14:41:04.391Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0" From 8aac3c9e8a323754eaf58b166c0959df1dddae79 Mon Sep 17 00:00:00 2001 From: owen Date: Sat, 15 Aug 2026 16:43:11 +0800 Subject: [PATCH 28/34] fix: harden cleaners, fix error masking, and remove dead code Cleaner base: - initialize origin_df/processed_df; read_data loads CSV with utf-8/gbk fallback - fix chained-assignment warning in streamflow anomaly_process via .loc - waterlevel: fix if/elif so moving_grad no longer warns; remove logger_level dead param Error handling: - streamflow_cleaner: drop try/except that masked KeyError as NameError (remove WTF print) - utils: minio_file_list propagates FS errors (graceful [] when S3 unconfigured) - utils: is_minio_folder normalizes s3:// prefix, handles empty dirs, propagates real errors Data source: - HydroData rejects empty uri; streamflow_unit adds 8D mapping and tolerates time_unit=None - remove 3 dead cargo-cult import pint_xarray lines (fixed full-suite pint flaky test) - dmca_esr: remove debug prints; scripts: migrate station_dataset_usage to uri= grdc: - replace assert with explicit ValueError (safe under -O); numeric (not lexicographic) id ordering - remove __main__ block that triggered full cache build; drop orphan get_local_root import Dead code (net -616 lines): - remove 11 unused utils unit-conversion helpers + 9 orphaned imports - delete unreferenced reader/minio_api.py module - remove orphan _rsvr_rolling_window_abrupt_abnormal_rm and deprecated voronoi functions - strip commented-out debug lines and garbage log call in rainfall_cleaner Tests: - add 8 regression test files (cleaner, error handling, nits, pint imports, unit mapping, dead code, suggestions, grdc ordering) - fix data_source one_day_dataset fixture and relax brittle compat assertion Deps: uv.lock synced for all extras (dev tooling) --- hydrodatasource/cleaner/cleaner.py | 16 +- hydrodatasource/cleaner/rainfall_cleaner.py | 6 - .../cleaner/rsvr_inflow_cleaner.py | 33 -- hydrodatasource/cleaner/streamflow_cleaner.py | 7 +- hydrodatasource/cleaner/waterlevel_cleaner.py | 2 +- .../processor/basin_mean_rainfall.py | 63 --- hydrodatasource/processor/dmca_esr.py | 3 - hydrodatasource/reader/data_source.py | 17 +- hydrodatasource/reader/gages.py | 2 - hydrodatasource/reader/grdc.py | 14 +- hydrodatasource/reader/minio_api.py | 140 ------- hydrodatasource/utils/utils.py | 381 ++---------------- scripts/station_dataset_usage.py | 6 +- tests/test_data_source.py | 2 +- tests/test_fix_cleaner_base.py | 141 +++++++ tests/test_fix_dead_code.py | 212 ++++++++++ tests/test_fix_error_handling.py | 164 ++++++++ tests/test_fix_grdc_ordering.py | 115 ++++++ tests/test_fix_nits.py | 115 ++++++ tests/test_fix_pint_imports.py | 121 ++++++ tests/test_fix_suggestions.py | 221 ++++++++++ tests/test_fix_unit_mapping.py | 119 ++++++ tests/test_hydroutils_compat.py | 4 +- uv.lock | 8 +- 24 files changed, 1273 insertions(+), 639 deletions(-) delete mode 100644 hydrodatasource/reader/minio_api.py create mode 100644 tests/test_fix_cleaner_base.py create mode 100644 tests/test_fix_dead_code.py create mode 100644 tests/test_fix_error_handling.py create mode 100644 tests/test_fix_grdc_ordering.py create mode 100644 tests/test_fix_nits.py create mode 100644 tests/test_fix_pint_imports.py create mode 100644 tests/test_fix_suggestions.py create mode 100644 tests/test_fix_unit_mapping.py diff --git a/hydrodatasource/cleaner/cleaner.py b/hydrodatasource/cleaner/cleaner.py index 6567e6f..95239a3 100644 --- a/hydrodatasource/cleaner/cleaner.py +++ b/hydrodatasource/cleaner/cleaner.py @@ -15,15 +15,29 @@ └── waterlevel_cleaner.py # 包含 WaterlevelCleaner 类 """ +import os + +import pandas as pd + class Cleaner: def __init__(self, data_folder, *args, **kwargs): self.data_path = data_folder + self.origin_df = None + self.processed_df = None self.read_data() def read_data(self): # 读取数据并存储在origin_df中 - pass + if self.data_path and os.path.isfile(self.data_path): + try: + self.origin_df = pd.read_csv(self.data_path) + except UnicodeDecodeError: + # 中文水文数据常为 GBK 编码 + self.origin_df = pd.read_csv(self.data_path, encoding="gbk") + else: + self.origin_df = pd.DataFrame() + self.processed_df = self.origin_df.copy() def save_data(self, data, output_path): # 保存数据到CSV diff --git a/hydrodatasource/cleaner/rainfall_cleaner.py b/hydrodatasource/cleaner/rainfall_cleaner.py index c03d395..c741583 100644 --- a/hydrodatasource/cleaner/rainfall_cleaner.py +++ b/hydrodatasource/cleaner/rainfall_cleaner.py @@ -280,7 +280,6 @@ def _station_yearly_sum(self, df_attr, df_station): df_station = pd.merge( df_station, df_attr[["STCD", "LGTD", "LTTD"]], on="STCD", how="left" ) - # self.logger.debug(df_station) return df_station def data_check_hourly_extreme( @@ -411,7 +410,6 @@ def data_check_time_series( ], } ) - # self.logger.debug(df_anomalies) if modify == True: # 遍历文件夹，清除对应行 folder_path = os.path.join(self.data_folder, basin_id) @@ -557,16 +555,13 @@ def __init__( output_folder=None, lower_bound=0, upper_bound=3000, - logger_level=logging.INFO, ): self.data_folder = data_folder self.output_folder = output_folder - self.logger_level = logger_level self.lower_bound = lower_bound self.upper_bound = upper_bound self.data_source_description = self.set_data_source_describe() self._check_file_format() - # self.station_info = self.read_site_info() def set_data_source_describe(self): data_source_dir = self.data_folder @@ -789,7 +784,6 @@ def process_basin(self, basin_id, filtered_data): valid_stations = self.process_stations(stations_df, basin) self.logger.debug(valid_stations["ID"]) valid_stations = valid_stations[valid_stations["ID"].isin(yearly_stations)] - self.logger.debug("11111111111111111111111111") self.logger.debug(valid_stations.head()) if valid_stations.empty: diff --git a/hydrodatasource/cleaner/rsvr_inflow_cleaner.py b/hydrodatasource/cleaner/rsvr_inflow_cleaner.py index b22dbc2..e5ea46c 100644 --- a/hydrodatasource/cleaner/rsvr_inflow_cleaner.py +++ b/hydrodatasource/cleaner/rsvr_inflow_cleaner.py @@ -159,39 +159,6 @@ def read_rsvr_info(self): ), f"Column {col} is not of numeric type" return rsvr_info - def _rsvr_rolling_window_abrupt_abnormal_rm( - self, df, var_col="RZ", threshold=50, window_size=5 - ): - """ - Detect and remove abnormal reservoir water level data using a rolling window. - - Parameters - ---------- - df : pd.DataFrame - The DataFrame containing the reservoir data. - var_col : str - the column to check, by default "RZ" - threshold: float - the threshold to remove the abnormal data - window_size : int - The size of the rolling window. - - Returns - ------- - pd.DataFrame - The DataFrame with an additional column indicating abnormal data. - """ - - # Calculate the median of the rolling window - df["median"] = df[var_col].rolling(window=window_size, center=True).median() - # Calculate the difference between the current value and the median - df["diff_median"] = abs(df[var_col] - df["median"]) - # Mark as abnormal if the difference exceeds the threshold - df["set_nan"] = df["diff_median"] > threshold - # Set abnormal values to NaN - df.loc[df["set_nan"], var_col] = np.nan - return df - def _rsvr_conservative_abrupt_abnormal_rm(self, df, var_col="RZ", threshold=10): """TODO: this method is not right, need to be fixed diff --git a/hydrodatasource/cleaner/streamflow_cleaner.py b/hydrodatasource/cleaner/streamflow_cleaner.py index 451fba4..14505ce 100644 --- a/hydrodatasource/cleaner/streamflow_cleaner.py +++ b/hydrodatasource/cleaner/streamflow_cleaner.py @@ -315,10 +315,7 @@ def adaptive_moving_average( end_date = date + pd.DateOffset(hours=half_window) # 计算窗口内的平均值 - try: - window_data = streamflow_data[start_date:end_date] - except KeyError: - print("WTF") + window_data = streamflow_data[start_date:end_date] smoothed_value = window_data.mean() smoothed_data.loc[date] = smoothed_value @@ -443,4 +440,4 @@ def anomaly_process(self, methods=None): self.processed_df[methods[0]] = streamflow_data # 去除提前插补的缺失值 - self.processed_df[methods[0]][self.origin_df["INQ"].isna()] = np.nan + self.processed_df.loc[self.origin_df["INQ"].isna(), methods[0]] = np.nan diff --git a/hydrodatasource/cleaner/waterlevel_cleaner.py b/hydrodatasource/cleaner/waterlevel_cleaner.py index d26fd26..7c705e0 100644 --- a/hydrodatasource/cleaner/waterlevel_cleaner.py +++ b/hydrodatasource/cleaner/waterlevel_cleaner.py @@ -84,7 +84,7 @@ def anomaly_process(self, methods=None): waterlevel_data = self.moving_gradient_filter( waterlevel_data=waterlevel_data ) - if method == "roll": + elif method == "roll": waterlevel_data = self.rolling_fill(waterlevel_data=waterlevel_data) else: print("please check your method name") diff --git a/hydrodatasource/processor/basin_mean_rainfall.py b/hydrodatasource/processor/basin_mean_rainfall.py index 5b909b4..4c6f2ef 100644 --- a/hydrodatasource/processor/basin_mean_rainfall.py +++ b/hydrodatasource/processor/basin_mean_rainfall.py @@ -1,10 +1,8 @@ import geopandas as gpd import numpy as np import pandas as pd -from matplotlib import pyplot as plt from scipy.spatial import Voronoi from shapely.geometry import Polygon -import hydrodatasource.configs.config as hdscc def calculate_thiesen_polygons(stations, basin): @@ -76,49 +74,6 @@ def calculate_thiesen_polygons(stations, basin): return clipped_polygons -def calculate_voronoi_polygons(stations, basin_geom): - """ - @deprecated - - Previous version of calculate_thiesen_polygons. - Deprecated in favor of calculate_thiesen_polygons. - Deprecated since version 0.0.11: Use calculate_thiesen_polygons instead. - - Parameters - ---------- - stations : GeoDataFrame - stations within the basin - basin_geom : GeoDataFrame - basin shapefile - - Returns - ------- - clipped_polygons_gdf : GeoDataFrame - clipped voronoi polygons - """ - - bounding_box = basin_geom.envelope.exterior.coords - points = np.array([point.coords[0] for point in stations.geometry]) - points_extended = np.concatenate((points, bounding_box)) - vor = Voronoi(points_extended) - regions = [vor.regions[vor.point_region[i]] for i in range(len(points))] - polygons = [ - Polygon(vor.vertices[region]).buffer(0) - for region in regions - if -1 not in region - ] - polygons_gdf = gpd.GeoDataFrame(geometry=polygons, crs=stations.crs) - polygons_gdf["station_id"] = stations["STCD"].astype(str).values - polygons_gdf["original_area"] = polygons_gdf.geometry.area - clipped_polygons_gdf = gpd.clip(polygons_gdf, basin_geom) - clipped_polygons_gdf["clipped_area"] = clipped_polygons_gdf.geometry.area - total_basin_area = basin_geom.area - clipped_polygons_gdf["area_ratio"] = ( - clipped_polygons_gdf["clipped_area"] / total_basin_area - ) - return clipped_polygons_gdf - - def calculate_weighted_rainfall( station_weights, rainfall_df, @@ -272,24 +227,6 @@ def basin_mean_func(df, weights_dict=None): return result -def plot_voronoi_polygons(original_polygons, clipped_polygons, basin): - fig, (ax_original, ax_clipped) = plt.subplots(1, 2, figsize=(12, 6)) - _plot_voronoi_polygons( - original_polygons, ax_original, basin, "Original Voronoi Polygons" - ) - _plot_voronoi_polygons( - clipped_polygons, ax_clipped, basin, "Clipped Voronoi Polygons" - ) - plt.tight_layout() - plt.show() - - -def _plot_voronoi_polygons(arg0, ax, basin, arg3): - arg0.plot(ax=ax, edgecolor="black") - basin.boundary.plot(ax=ax, color="red") - ax.set_title(arg3) - - def stations_within_basin(basin_gdf, station_gdf, buffer_m=0, basin_crs_epsg=3857): """ Get stations within the buffered basin boundary diff --git a/hydrodatasource/processor/dmca_esr.py b/hydrodatasource/processor/dmca_esr.py index dabba8d..5149933 100644 --- a/hydrodatasource/processor/dmca_esr.py +++ b/hydrodatasource/processor/dmca_esr.py @@ -746,8 +746,6 @@ def rainfall_runoff_event_identify( flow_threshold, multiple=multiple, ) - print(beginning_rain, end_rain, beginning_flow, end_flow) - print(len(beginning_rain), len(end_rain), len(beginning_flow), len(end_flow)) return pd.DataFrame( { "BEGINNING_RAIN": beginning_rain, @@ -815,7 +813,6 @@ def get_rr_events(rain, flow, basin_area, max_window=100, max_flow_min=None): multiple = num * 24 else: raise ValueError(f"Unsupported unit: {unit}") - print(f"flow.units = { flow.units}, multiple = {multiple}") rr_events = {} for i, basin in enumerate(basin_area.basin.values): rr_event = rainfall_runoff_event_identify( diff --git a/hydrodatasource/reader/data_source.py b/hydrodatasource/reader/data_source.py index 2bf91b4..3046efe 100644 --- a/hydrodatasource/reader/data_source.py +++ b/hydrodatasource/reader/data_source.py @@ -44,7 +44,7 @@ class HydroData(ABC): """ def __init__(self, uri): - if uri is None: + if not uri: raise ValueError( "uri must be provided: pass an absolute path or s3:// URI " "pointing to the data directory." @@ -118,8 +118,14 @@ def __init__( @property def streamflow_unit(self): - unit_mapping = {"1h": "mm/h", "3h": "mm/3h", "1D": "mm/d", "1M": "mm/M"} - return {unit: unit_mapping[unit] for unit in self.time_unit} + unit_mapping = { + "1h": "mm/h", + "3h": "mm/3h", + "1D": "mm/d", + "8D": "mm/8d", + "1M": "mm/M", + } + return {unit: unit_mapping[unit] for unit in (self.time_unit or [])} def get_name(self): return "SelfMadeHydroDataset" @@ -506,9 +512,6 @@ def cache_attributes_xrdataset(self): ------- None """ - # NOTICE: although it seems that we don't use pint_xarray, we have to import this package - import pint_xarray # noqa: F401 - df_attr = self.read_attributes() df_attr.set_index("basin_id", inplace=True) @@ -2783,8 +2786,6 @@ def read_attr_xrdataset(self, gage_id_lst=None, var_lst=None, **kwargs): def cache_intermediate_attributes_xrdataset(self): """Convert intermediate attributes to a single dataset and cache as NetCDF.""" - import pint_xarray # noqa: F401 - # Resolve shape file (prefer intermediate, else base) shape_file_inter = self._get_intermediate_shape_file() if os.path.exists(shape_file_inter): diff --git a/hydrodatasource/reader/gages.py b/hydrodatasource/reader/gages.py index 7d1261c..706f862 100644 --- a/hydrodatasource/reader/gages.py +++ b/hydrodatasource/reader/gages.py @@ -653,8 +653,6 @@ def cache_attributes_xrdataset(self): ------- None """ - # NOTICE: although it seems that we don't use pint_xarray, we have to import this package - import pint_xarray # noqa: F401 from hydrodatasource.configs import config as conf # 1. Get all site IDs diff --git a/hydrodatasource/reader/grdc.py b/hydrodatasource/reader/grdc.py index 0a8d392..552a78f 100644 --- a/hydrodatasource/reader/grdc.py +++ b/hydrodatasource/reader/grdc.py @@ -21,7 +21,7 @@ import geopandas as gpd from dateutil.parser import parse -from hydrodatasource.configs.config import CACHE_DIR, get_local_root +from hydrodatasource.configs.config import CACHE_DIR from hydrodatasource.reader.data_source import HydroData @@ -85,7 +85,9 @@ def read_site_info(self): gdf["grdc_no"] = gdf["grdc_no"].apply( lambda x: str(int(x)) if isinstance(x, float) else str(x) ) - assert all(x < y for x, y in zip(gdf["grdc_no"], gdf["grdc_no"][1:])) + grdc_ids = gdf["grdc_no"].astype(int) + if not all(x < y for x, y in zip(grdc_ids, grdc_ids[1:])): + raise ValueError("grdc_no column must be sorted in ascending order") return gdf[["grdc_no", "area"]] def map_station_to_continent(self, station_id: str): @@ -606,11 +608,3 @@ def _convert_to_serializable(obj): elif isinstance(obj, list): return [_convert_to_serializable(i) for i in obj] return obj - - -if __name__ == "__main__": - data_dir = str(get_local_root() / "datasets-origin" / "GRDC") - grdc = Grdc(data_dir) - # grdc.cache_grdc_daily(["1107700", "4101200"], ["1990-10-01", "2000-10-01"]) - grdc.cache_grdc_daily() - grdc.read_streamflow_xrdataset(["1107700", "4101200"], ["1990-10-01", "2000-10-01"]) diff --git a/hydrodatasource/reader/minio_api.py b/hydrodatasource/reader/minio_api.py deleted file mode 100644 index 076c626..0000000 --- a/hydrodatasource/reader/minio_api.py +++ /dev/null @@ -1,140 +0,0 @@ -"""Main module.""" -import asyncio -import os -import pathlib - -from minio import Minio -import chardet - - -async def minio_upload_csv(client, bucket_name, object_name, file_path): - """upload csv to minio - - Parameters - ---------- - client : _type_ - the minio client - bucket_name : _type_ - the bucket name - object_name : _type_ - the object name - file_path : _type_ - the local file path - """ - # Make a bucket - bucket_names = [bucket.name for bucket in client.list_buckets()] - if bucket_name not in bucket_names: - client.make_bucket(bucket_name) - # Upload an object - client.fput_object(bucket_name, object_name, file_path) - # List objects - objects = client.list_objects(bucket_name, recursive=True) - return [obj.object_name for obj in objects] - - -async def minio_download_csv(client: Minio, bucket_name, object_name: str, file_path: str, version_id=None): - try: - response = client.get_object(bucket_name, object_name, version_id) - # 图片不能直接解码, 但可以直接对文件写入response.data - encoding = chardet.detect(response.data)['encoding'] - object_path = os.path.join(file_path, object_name) - object_parent = pathlib.Path(object_path).parent - if not object_parent.exists(): - object_parent.mkdir(parents=True) - if encoding is not None: - res_csv: str = response.data.decode(encoding) - with open(object_path, 'w+', encoding=encoding) as fp: - fp.write(res_csv) - else: - with open(object_path, 'wb') as fp: - fp.write(response.data) - finally: - response.close() - response.release_conn() - - -async def boto3_upload_csv(client, bucket_name, object_name, file_path): - """upload csv to minio - - Parameters - ---------- - client : _type_ - the minio client - bucket_name : _type_ - the bucket name - object_name : _type_ - the object name - file_path : _type_ - the local file path - """ - # Make a bucket - bucket_names = [dic['Name'] for dic in client.list_buckets()['Buckets']] - if bucket_name not in bucket_names: - client.create_bucket(Bucket=bucket_name) - # Upload an object - client.upload_file(file_path, bucket_name, object_name) - # List objects - objects = [dic['Key'] for dic in client.list_objects(Bucket=bucket_name)['Contents']] - return objects - - -async def boto3_download_csv(client, bucket_name, object_name, file_path: str): - client.download_file(bucket_name, object_name, file_path) - - -async def boto3_sync_files(client, bucket_name, local_path, bucket_path=None): - """ - :param client: the boto3 client - :param bucket_name: the bucket name which you want to sync your data - :param local_path: the path on your local machine - :param bucket_path: the path under your bucket which you want to sync - :return: - """ - remote_objects = [dic['Key'] for dic in client.list_objects(Bucket=bucket_name)['Contents']] - local_objects = [str(path.relative_to(local_path)).replace('\\', '/') - for path in pathlib.Path(local_path).rglob(pattern='*') if path.is_file()] - objects_in_remote = [obj for obj in remote_objects if obj not in local_objects] - objects_in_local = [obj for obj in local_objects if obj not in remote_objects] - task_batch_dload = asyncio.create_task(boto3_batch_download(objects_in_remote, client, bucket_name, local_path)) - task_batch_upload = asyncio.create_task(boto3_batch_upload(objects_in_local, client, bucket_name, local_path)) - await asyncio.gather(task_batch_upload, task_batch_dload) - - -async def minio_batch_download(objects_in_remote, client: Minio, bucket_name, local_path): - # 将下载任务打包成大量task容易造成许多上下文切换进而挤兑，目前看来不如顺序下载 - for obj in objects_in_remote: - await minio_download_csv(client, bucket_name, obj, local_path) - - -async def minio_batch_upload(objects_in_local, client: Minio, bucket_name, local_path): - for obj in objects_in_local: - await minio_upload_csv(client, bucket_name, obj, local_path) - - -async def boto3_batch_download(objects_in_remote, client, bucket_name, local_path): - # 将下载任务打包成大量task容易造成许多上下文切换进而挤兑，目前看来不如顺序下载 - for obj in objects_in_remote: - await boto3_download_csv(client, bucket_name, obj, local_path) - - -async def boto3_batch_upload(objects_in_remote, client, bucket_name, local_path): - for obj in objects_in_remote: - await boto3_upload_csv(client, bucket_name, obj, local_path) - - -async def minio_sync_files(client: Minio, bucket_name, local_path, bucket_path=None): - """ - :param client: the minio client - :param bucket_name: the bucket name which you want to sync your data - :param local_path: the path on your local machine, contents of bucket_name will be saved to the directory directly - :param bucket_path: the path under your bucket which you want to sync - :return: - """ - remote_objects = [obj.object_name for obj in client.list_objects(bucket_name, prefix=bucket_path, recursive=True)] - local_objects = [str(path.relative_to(local_path)).replace('\\', '/') - for path in pathlib.Path(local_path).rglob(pattern='*') if path.is_file()] - objects_in_remote = [obj for obj in remote_objects if obj not in local_objects] - objects_in_local = [obj for obj in local_objects if obj not in remote_objects] - task_batch_dload = asyncio.create_task(minio_batch_download(objects_in_remote, client, bucket_name, local_path)) - task_batch_upload = asyncio.create_task(minio_batch_upload(objects_in_local, client, bucket_name, local_path)) - await asyncio.gather(task_batch_upload, task_batch_dload) diff --git a/hydrodatasource/utils/utils.py b/hydrodatasource/utils/utils.py index 55b4de5..f24dea2 100644 --- a/hydrodatasource/utils/utils.py +++ b/hydrodatasource/utils/utils.py @@ -1,128 +1,11 @@ -import re -import numpy as np -import geopandas as gpd -from netCDF4 import Dataset, date2num, num2date -import time -from datetime import datetime, timedelta -import pint -import xarray as xr import contextlib import tempfile -from ..configs.config import FS - -from hydroutils.hydro_time import calculate_utc_offset - - -def creatspinc(value, data_vars, lats, lons, starttime, filename, resolution): - gridspi = Dataset(filename, "w", format="NETCDF4") - - # dimensions - gridspi.createDimension("time", value[0].shape[0]) - gridspi.createDimension("lat", value[0].shape[2]) # len(lat) - gridspi.createDimension("lon", value[0].shape[1]) - - # Create coordinate variables for dimensions - times = gridspi.createVariable("time", np.float64, ("time",)) - latitudes = gridspi.createVariable("lat", np.float32, ("lat",)) - longitudes = gridspi.createVariable("lon", np.float32, ("lon",)) - - # Create the actual variable - for var, attr in data_vars.items(): - gridspi.createVariable( - var, - np.float32, - ( - "time", - "lon", - "lat", - ), - ) - - # Global Attributes - gridspi.description = "var" - gridspi.history = f"Created {time.ctime(time.time())}" - gridspi.source = "netCDF4 python module tutorial" - - # Variable Attributes - latitudes.units = "degree_north" - longitudes.units = "degree_east" - times.units = "days since 1970-01-01 00:00:00" - times.calendar = "gregorian" - - # data - latitudes[:] = lats - longitudes[:] = lons - - # Fill in times - dates = [] - if resolution == "daily": - for n in range(value[0].shape[0]): - dates.append(starttime + n) - times[:] = dates[:] - - elif resolution == "6-hourly": - # for n in range(value[0].shape[0]): - # dates.append(starttime + (n+1) * np.timedelta64(6, 'h')) - - for n in range(value[0].shape[0]): - dates.append(starttime + (n + 1) * timedelta(hours=6)) - - times[:] = date2num(dates, units=times.units, calendar=times.calendar) - # print 'time values (in units %s): ' % times.units +'\n', times[:] - dates = num2date(times[:], units=times.units, calendar=times.calendar) - - # Fill in values - i = 0 - for var, attr in data_vars.items(): - gridspi.variables[var].long_name = attr["long_name"] - gridspi.variables[var].units = attr["units"] - gridspi.variables[var][:] = value[i][:] - i = i + 1 - - gridspi.close() - - -def regen_box(bbox, resolution, offset): - lx = bbox[0] - rx = bbox[2] - LLON = np.round( - int(lx) - + resolution * int((lx - int(lx)) / resolution + 0.5) - + offset - * (int(lx * 10) / 10 + offset - lx) - / abs(int(lx * 10) // 10 + offset - lx + 0.0000001), - 3, - ) - RLON = np.round( - int(rx) - + resolution * int((rx - int(rx)) / resolution + 0.5) - - offset - * (int(rx * 10) / 10 + offset - rx) - / abs(int(rx * 10) // 10 + offset - rx + 0.0000001), - 3, - ) +from datetime import datetime - by = bbox[1] - ty = bbox[3] - BLAT = np.round( - int(by) - + resolution * int((by - int(by)) / resolution + 0.5) - + offset - * (int(by * 10) / 10 + offset - by) - / abs(int(by * 10) // 10 + offset - by + 0.0000001), - 3, - ) - TLAT = np.round( - int(ty) - + resolution * int((ty - int(ty)) / resolution + 0.5) - - offset - * (int(ty * 10) / 10 + offset - ty) - / abs(int(ty * 10) // 10 + offset - ty + 0.0000001), - 3, - ) +import geopandas as gpd - # print(LLON,BLAT,RLON,TLAT) - return [LLON, BLAT, RLON, TLAT] +from ..configs.config import FS +from hydroutils.hydro_time import calculate_utc_offset def validate(date_text, formatter, error): @@ -132,222 +15,6 @@ def validate(date_text, formatter, error): raise ValueError(error) from e -def cf2datetime(ds): - ds = ds.copy() - time_tmp1 = ds.indexes["time"] - attrs = ds.coords["time"].attrs - time_tmp2 = [] - for i in range(time_tmp1.shape[0]): - tmp = time_tmp1[i] - a = str(tmp.year).zfill(4) - b = str(tmp.month).zfill(2) - c = str(tmp.day).zfill(2) - d = str(tmp.hour).zfill(2) - e = str(tmp.minute).zfill(2) - f = str(tmp.second).zfill(2) - time_tmp2.append(np.datetime64(f"{a}-{b}-{c} {d}:{e}:{f}.00000000")) - ds = ds.assign_coords(time=time_tmp2) - ds.coords["time"].attrs = attrs - - return ds - - -def generate_time_intervals(start_date, end_date): - # Initialize an empty list to store the intervals - intervals = [] - - # Loop over days - while start_date <= end_date: - # Loop over the four time intervals in a day - intervals.extend( - [start_date.strftime("%Y-%m-%d"), hour] for hour in ["00", "06", "12", "18"] - ) - # Move to the next day - start_date += timedelta(days=1) - - return intervals - - -def _convert_target_unit(target_unit): - """Convert user-friendly unit to standard unit for internal calculations.""" - if match := re.match(r"mm/(\d+)(h|d)", target_unit): - num, unit = match.groups() - return int(num), unit - return None, None - - -def _process_custom_unit(streamflow_data, custom_unit): - """Process streamflow data with custom unit format like mm/3h.""" - custom_unit_pattern = re.compile(r"mm/(\d+)(h|d)") - if custom_match := custom_unit_pattern.match(custom_unit): - num, unit = custom_match.groups() - if unit == "h": - standard_unit = "mm/h" - conversion_factor = int(num) - elif unit == "d": - standard_unit = "mm/d" - conversion_factor = int(num) - else: - raise ValueError(f"Unsupported unit: {unit}") - - # Convert custom unit to standard unit - if isinstance(streamflow_data, xr.Dataset): - # For xarray, modify the data and attributes - result = streamflow_data / conversion_factor - result[list(result.keys())[0]].attrs["units"] = standard_unit - return result - else: - # For numpy/pandas, just return the converted values - return streamflow_data / conversion_factor, standard_unit - else: - # If it's not a custom unit format, return as is - if isinstance(streamflow_data, xr.Dataset): - result = streamflow_data.copy() - result[list(result.keys())[0]].attrs["units"] = custom_unit - return result - else: - return streamflow_data, custom_unit - - -def _get_unit_conversion_info(unit_str): - """Get conversion information for a unit string. - - Returns: - tuple: (standard_unit, conversion_factor) where conversion_factor - is used to convert from standard unit to custom unit. - """ - if not (match := re.match(r"mm/(\d+)(h|d)", unit_str)): - # For standard units, no conversion needed - return unit_str, 1 - num, unit = match.groups() - if unit == "h": - return "mm/h", int(num) - elif unit == "d": - return "mm/d", int(num) - else: - raise ValueError(f"Unsupported unit: {unit}") - - -def _get_actual_source_unit(streamflow_data, source_unit=None): - """Determine the actual source unit from streamflow data. - - Parameters - ---------- - streamflow_data : xarray.Dataset, pint.Quantity, numpy.ndarray, - pandas.DataFrame/Series - The streamflow data to extract units from - source_unit : str, optional - Explicitly provided source unit that overrides data units - - Returns - ------- - str or None - The actual source unit string, or None if no unit information found - """ - if source_unit is not None: - return source_unit - - if isinstance(streamflow_data, xr.Dataset): - streamflow_key = list(streamflow_data.keys())[0] - # First check attrs for units - if "units" in streamflow_data[streamflow_key].attrs: - return streamflow_data[streamflow_key].attrs["units"] - # Then check if it has pint units - try: - return str(streamflow_data[streamflow_key].pint.units) - except (AttributeError, ValueError): - return None - elif isinstance(streamflow_data, pint.Quantity): - return str(streamflow_data.units) - else: - # numpy array or pandas without units - return None - - -def _normalize_unit(unit_str): - """Normalize unit string for comparison (handle m3/s vs m^3/s and pint format).""" - if not unit_str: - return unit_str - - # Handle pint verbose format - normalized = unit_str.replace("meter ** 3 / second", "m^3/s") - normalized = normalized.replace("meter**3/second", "m^3/s") - normalized = normalized.replace("cubic_meter / second", "m^3/s") - normalized = normalized.replace("cubic_meter/second", "m^3/s") - - # Handle short format variations - normalized = normalized.replace("m3/s", "m^3/s") - normalized = normalized.replace("ft3/s", "ft^3/s") - normalized = normalized.replace("ft**3/s", "ft^3/s") - normalized = normalized.replace("cubic_foot / second", "ft^3/s") - normalized = normalized.replace("cubic_foot/second", "ft^3/s") - - # Handle pint format for depth units - normalized = normalized.replace("millimeter / day", "mm/d") - normalized = normalized.replace("millimeter/day", "mm/d") - normalized = normalized.replace("millimeter / hour", "mm/h") - normalized = normalized.replace("millimeter/hour", "mm/h") - - return normalized - - -def _is_inverse_conversion(source_unit, target_unit): - """Determine if this should be an inverse conversion based on units. - - Returns True if converting from depth units (mm/time) to volume units - (m^3/s). - Returns False if converting from volume units to depth units. - """ - source_norm = _normalize_unit(source_unit) if source_unit else "" - target_norm = _normalize_unit(target_unit) - - # Define unit patterns - depth_pattern = re.compile(r"mm/(?:\d+)?[hd]?(?:ay|our)?$") - volume_pattern = re.compile(r"(?:m\^?3|ft\^?3)/s$") - - source_is_depth = bool(depth_pattern.match(source_norm)) - source_is_volume = bool(volume_pattern.match(source_norm)) - target_is_depth = bool(depth_pattern.match(target_norm)) - target_is_volume = bool(volume_pattern.match(target_norm)) - - if source_is_depth and target_is_volume: - return True - elif source_is_volume and target_is_depth: - return False - else: - # If we can't determine from units, return None to indicate ambiguity - return None - - -def _validate_inverse_consistency(source_unit, target_unit, inverse_param): - """Validate that the inverse parameter is consistent with the units. - - Parameters - ---------- - source_unit : str - Source unit string - target_unit : str - Target unit string - inverse_param : bool - The inverse parameter provided by user - - Raises - ------ - ValueError - If inverse parameter is inconsistent with unit conversion direction - """ - expected_inverse = _is_inverse_conversion(source_unit, target_unit) - - if expected_inverse is not None and expected_inverse != inverse_param: - direction = "depth->volume" if expected_inverse else "volume->depth" - raise ValueError( - f"Inverse parameter ({inverse_param}) is inconsistent with unit " - f"conversion direction. Converting from '{source_unit}' to " - f"'{target_unit}' suggests {direction} conversion " - f"(inverse={expected_inverse})." - ) - - def streamflow_unit_conv( streamflow, area, @@ -443,14 +110,13 @@ def minio_file_list(minio_folder_url): folder list """ # Get the list of files in the directory - try: - # the minio folder url doesn't have to start with s3://, but we agree that it must - # start with s3:// to distinguish between local and Minio folder directories. - files = FS.ls(minio_folder_url) - return [file.split("/")[-1] for file in files if not file.endswith("/")] - except Exception as e: - print(f"Error accessing {minio_folder_url}: {e}") + # the minio folder url doesn't have to start with s3://, but we agree that it must + # start with s3:// to distinguish between local and Minio folder directories. + if FS is None: + # S3 not configured in this environment; nothing to list return [] + files = FS.ls(minio_folder_url) + return [file.split("/")[-1] for file in files if not file.endswith("/")] def is_minio_folder(minio_url): @@ -468,18 +134,21 @@ def is_minio_folder(minio_url): True if the folder exists, False otherwise """ - try: - if not FS.exists(minio_url): - raise FileNotFoundError(f"No file or folder found in {minio_url}") - if minio_url.endswith("/"): - # If the path ends with '/', treat it as a directory - return True - # Try to list objects under this path - objects = FS.ls(minio_url) - test_object = "s3://" + objects[0] - return len(objects) != 1 or test_object != minio_url - except Exception as e: - raise NotImplementedError(f"Error accessing {minio_url}: {e}") from e + if not FS.exists(minio_url): + raise FileNotFoundError(f"No file or folder found in {minio_url}") + if minio_url.endswith("/"): + # If the path ends with '/', treat it as a directory + return True + # Try to list objects under this path + objects = FS.ls(minio_url) + # 存在但无子项 → 目录 + if not objects: + return True + # 归一化比较（s3fs ls 结果可能带或不带 s3:// 前缀） + target = minio_url.rstrip("/").removeprefix("s3://") + single = str(objects[0]).rstrip("/").removeprefix("s3://") + # 只有自身一个子项 → 是文件；否则是目录 + return len(objects) != 1 or single != target def calculate_basin_offsets(shp_file_path): diff --git a/scripts/station_dataset_usage.py b/scripts/station_dataset_usage.py index 0b84649..dea7431 100644 --- a/scripts/station_dataset_usage.py +++ b/scripts/station_dataset_usage.py @@ -23,9 +23,8 @@ def main(): # Create StationHydroDataset instance dataset = StationHydroDataset( - data_path=data_path, + uri=data_path, time_unit=["1D", "3h"], # Support both daily and 3-hourly data - dataset_name="songliao_station_dataset", offset_to_utc=False, # Convert Beijing time to UTC ) @@ -163,9 +162,8 @@ def demonstrate_basin_analysis(): data_path = "/mnt/data/ClassC/songliaorrevent" dataset = StationHydroDataset( - data_path=data_path, + uri=data_path, time_unit=["1D"], - dataset_name="basin_analysis", version="v1.0", ) diff --git a/tests/test_data_source.py b/tests/test_data_source.py index ab12a03..7369159 100644 --- a/tests/test_data_source.py +++ b/tests/test_data_source.py @@ -43,7 +43,7 @@ def three_hour_dataset(): def one_day_dataset(): if not _SELFMADE_DATA: pytest.skip("Set HDS_SELFMADE_DATA env var") - return SelfMadeHydroDataset(uri=_SELFMADE_DATA) + return SelfMadeHydroDataset(uri=_SELFMADE_DATA, time_unit=["1D"]) @pytest.fixture diff --git a/tests/test_fix_cleaner_base.py b/tests/test_fix_cleaner_base.py new file mode 100644 index 0000000..6a6f670 --- /dev/null +++ b/tests/test_fix_cleaner_base.py @@ -0,0 +1,141 @@ +"""Regression tests for the Cleaner base-class bug. + +Bug (review finding): ``Cleaner.__init__`` (hydrodatasource/cleaner/cleaner.py) +never initializes ``origin_df`` / ``processed_df``, and ``read_data()`` is a +no-op (``pass``) even though its docstring claims it "reads data and stores it +in origin_df". As a result: + + * ``StreamflowCleaner.anomaly_process`` (streamflow_cleaner.py:393-407) + reads ``self.origin_df`` and writes ``self.processed_df`` + * ``WaterlevelCleaner.anomaly_process`` (waterlevel_cleaner.py:79-94) + does the same + +...so every call raises ``AttributeError`` and both cleaner classes are +effectively dead. The documented usage is +``scripts/chinese_rsvr_inflow_preprocessing.py:41-53`` — construct a +``StreamflowCleaner`` with a CSV path, then call ``anomaly_process(["EMA"])``. + +Contract pinned here (implementation-agnostic, no real data required): + * After construction a cleaner exposes ``origin_df`` and ``processed_df`` + as pandas DataFrames. + * Given a minimal CSV with the columns the methods expect, ``anomaly_process`` + completes without raising ``AttributeError`` and stores the cleaned series + back onto ``processed_df``. + +RED on the current (unfixed) code: construction exposes no such attributes and +``anomaly_process`` raises ``AttributeError`` immediately. +""" + + +def _write_streamflow_csv(tmp_path): + """Write a minimal streamflow CSV (TM + INQ columns) and return its path.""" + import pandas as pd + + csv_path = tmp_path / "streamflow_input.csv" + df = pd.DataFrame( + { + "TM": pd.date_range("2023-01-01", periods=48, freq="D"), + "INQ": [100, 110, 120, 130, 95, 105, 115, 125] * 6, + } + ) + df.to_csv(csv_path, index=False) + return csv_path + + +def _write_waterlevel_csv(tmp_path): + """Write a minimal waterlevel CSV (TM + Z columns) and return its path.""" + import pandas as pd + + csv_path = tmp_path / "waterlevel_input.csv" + df = pd.DataFrame( + { + "TM": pd.date_range("2023-01-01", periods=48, freq="D"), + "Z": [100.0, 100.5, 101.0, 100.8, 101.2, 101.5, 101.3, 101.8] * 6, + } + ) + df.to_csv(csv_path, index=False) + return csv_path + + +def test_cleaner_base_exposes_origin_and_processed_df_after_construction(tmp_path): + """The base Cleaner exposes origin_df / processed_df after construction.""" + import pandas as pd + + from hydrodatasource.cleaner.cleaner import Cleaner + + csv_path = _write_streamflow_csv(tmp_path) + cleaner = Cleaner(data_folder=str(csv_path)) + + assert isinstance(cleaner.origin_df, pd.DataFrame) + assert isinstance(cleaner.processed_df, pd.DataFrame) + + +def test_streamflow_cleaner_exposes_origin_and_processed_df_after_construction( + tmp_path, +): + """StreamflowCleaner exposes origin_df / processed_df after construction.""" + import pandas as pd + + from hydrodatasource.cleaner.streamflow_cleaner import StreamflowCleaner + + csv_path = _write_streamflow_csv(tmp_path) + cleaner = StreamflowCleaner(data_folder=str(csv_path)) + + assert isinstance(cleaner.origin_df, pd.DataFrame) + assert isinstance(cleaner.processed_df, pd.DataFrame) + + +def test_streamflow_cleaner_anomaly_process_moving_average_writes_processed_df( + tmp_path, +): + """anomaly_process(['moving_average']) runs without AttributeError.""" + from hydrodatasource.cleaner.streamflow_cleaner import StreamflowCleaner + + csv_path = _write_streamflow_csv(tmp_path) + cleaner = StreamflowCleaner(data_folder=str(csv_path)) + + cleaner.anomaly_process(["moving_average"]) + + assert "moving_average" in cleaner.processed_df.columns + assert len(cleaner.processed_df) == 48 + + +def test_streamflow_cleaner_anomaly_process_ema_writes_processed_df(tmp_path): + """anomaly_process(['EMA']) — the documented usage — runs without AttributeError.""" + from hydrodatasource.cleaner.streamflow_cleaner import StreamflowCleaner + + csv_path = _write_streamflow_csv(tmp_path) + cleaner = StreamflowCleaner(data_folder=str(csv_path)) + + cleaner.anomaly_process(["EMA"]) + + assert "EMA" in cleaner.processed_df.columns + assert len(cleaner.processed_df) == 48 + + +def test_waterlevel_cleaner_exposes_origin_and_processed_df_after_construction( + tmp_path, +): + """WaterlevelCleaner exposes origin_df / processed_df after construction.""" + import pandas as pd + + from hydrodatasource.cleaner.waterlevel_cleaner import WaterlevelCleaner + + csv_path = _write_waterlevel_csv(tmp_path) + cleaner = WaterlevelCleaner(data_folder=str(csv_path)) + + assert isinstance(cleaner.origin_df, pd.DataFrame) + assert isinstance(cleaner.processed_df, pd.DataFrame) + + +def test_waterlevel_cleaner_anomaly_process_roll_writes_processed_df(tmp_path): + """anomaly_process(['roll']) runs without AttributeError and writes results.""" + from hydrodatasource.cleaner.waterlevel_cleaner import WaterlevelCleaner + + csv_path = _write_waterlevel_csv(tmp_path) + cleaner = WaterlevelCleaner(data_folder=str(csv_path)) + + cleaner.anomaly_process(["roll"]) + + assert len(cleaner.processed_df.columns) >= 1 + assert len(cleaner.processed_df) == 48 diff --git a/tests/test_fix_dead_code.py b/tests/test_fix_dead_code.py new file mode 100644 index 0000000..aa454f1 --- /dev/null +++ b/tests/test_fix_dead_code.py @@ -0,0 +1,212 @@ +"""Guard tests: previously-identified dead code must stay deleted. + +Purpose +------- +A prior review pass flagged a batch of dead-code candidates (functions/modules +with no callers anywhere in the repository). The implementer deletes them. These +tests guard the invariant "the dead symbols no longer appear in the source tree". + +These are source-inspection tests (same style as ``test_fix_pint_imports.py``): +they read the relevant ``.py`` file and assert the symbol's definition line is +absent. They deliberately avoid marking ``internal_data`` and never import the +package at module scope, so collection is side-effect free. + +Candidate inventory +------------------- +A. ``hydrodatasource/utils/utils.py`` dead helper functions (superseded by the + hydroutils wrapper ``streamflow_unit_conv``, which imports its helpers from + ``hydroutils.hydro_units``, not from the local module): + creatspinc, regen_box, cf2datetime, generate_time_intervals, + _convert_target_unit, _process_custom_unit, _get_unit_conversion_info, + _get_actual_source_unit, _normalize_unit, _is_inverse_conversion, + _validate_inverse_consistency + The last three only reference each other; none is referenced by the active + functions ``streamflow_unit_conv`` / ``minio_file_list`` / ``is_minio_folder`` / + ``calculate_basin_offsets`` / ``cal_area_from_shp``. + +B. ``hydrodatasource/reader/minio_api.py``: whole module, never imported. + +C. ``hydrodatasource/cleaner/rsvr_inflow_cleaner.py``: + ``_rsvr_rolling_window_abrupt_abnormal_rm``, never called. + +D. ``hydrodatasource/processor/basin_mean_rainfall.py``: + ``calculate_voronoi_polygons``, ``plot_voronoi_polygons``, + ``_plot_voronoi_polygons`` (the active functions ``calculate_weighted_rainfall`` + and ``calculate_thiesen_polygons`` are untouched). +""" + +from pathlib import Path + +# Paths are resolved at import time from this file's location; this does not +# import the package, so it is collection-safe. +_REPO_ROOT = Path(__file__).resolve().parents[1] +_UTILS_PATH = _REPO_ROOT / "hydrodatasource" / "utils" / "utils.py" +_MINIO_API_PATH = _REPO_ROOT / "hydrodatasource" / "reader" / "minio_api.py" +_RSVR_CLEANER_PATH = ( + _REPO_ROOT / "hydrodatasource" / "cleaner" / "rsvr_inflow_cleaner.py" +) +_BASIN_MEAN_PATH = ( + _REPO_ROOT / "hydrodatasource" / "processor" / "basin_mean_rainfall.py" +) + + +def _source(path: Path) -> str: + """Return the current UTF-8 text of a source file.""" + return path.read_text(encoding="utf-8") + + +# --------------------------------------------------------------------------- +# A. hydrodatasource/utils/utils.py — dead helper functions +# --------------------------------------------------------------------------- + + +def test_utils_creatspinc_removed(): + """The dead netCDF grid-writing helper ``creatspinc`` must be gone.""" + assert "def creatspinc" not in _source(_UTILS_PATH), ( + "hydrodatasource/utils/utils.py still defines creatspinc, which has no " + "callers anywhere in the repository." + ) + + +def test_utils_regen_box_removed(): + """The dead bbox-realignment helper ``regen_box`` must be gone.""" + assert "def regen_box" not in _source(_UTILS_PATH), ( + "hydrodatasource/utils/utils.py still defines regen_box, which has no " + "callers anywhere in the repository." + ) + + +def test_utils_cf2datetime_removed(): + """The dead CF-time-coordinate helper ``cf2datetime`` must be gone.""" + assert "def cf2datetime" not in _source(_UTILS_PATH), ( + "hydrodatasource/utils/utils.py still defines cf2datetime, which has no " + "callers anywhere in the repository." + ) + + +def test_utils_generate_time_intervals_removed(): + """The dead time-interval generator ``generate_time_intervals`` must be gone.""" + assert "def generate_time_intervals" not in _source(_UTILS_PATH), ( + "hydrodatasource/utils/utils.py still defines generate_time_intervals, " + "which has no callers anywhere in the repository." + ) + + +def test_utils__convert_target_unit_removed(): + """The dead unit-parsing helper ``_convert_target_unit`` must be gone.""" + assert "def _convert_target_unit" not in _source(_UTILS_PATH), ( + "hydrodatasource/utils/utils.py still defines _convert_target_unit, which " + "has no callers anywhere in the repository." + ) + + +def test_utils__process_custom_unit_removed(): + """The dead custom-unit helper ``_process_custom_unit`` must be gone.""" + assert "def _process_custom_unit" not in _source(_UTILS_PATH), ( + "hydrodatasource/utils/utils.py still defines _process_custom_unit, which " + "has no callers anywhere in the repository." + ) + + +def test_utils__get_unit_conversion_info_removed(): + """The dead conversion-info helper ``_get_unit_conversion_info`` must be gone.""" + assert "def _get_unit_conversion_info" not in _source(_UTILS_PATH), ( + "hydrodatasource/utils/utils.py still defines _get_unit_conversion_info, " + "which has no callers anywhere in the repository." + ) + + +def test_utils__get_actual_source_unit_removed(): + """The dead source-unit helper ``_get_actual_source_unit`` must be gone.""" + assert "def _get_actual_source_unit" not in _source(_UTILS_PATH), ( + "hydrodatasource/utils/utils.py still defines _get_actual_source_unit, " + "which has no callers anywhere in the repository." + ) + + +def test_utils__normalize_unit_removed(): + """The dead unit-normalization helper ``_normalize_unit`` must be gone.""" + assert "def _normalize_unit" not in _source(_UTILS_PATH), ( + "hydrodatasource/utils/utils.py still defines _normalize_unit, which is " + "only referenced by the dead _is_inverse_conversion helper." + ) + + +def test_utils__is_inverse_conversion_removed(): + """The dead inverse-direction helper ``_is_inverse_conversion`` must be gone.""" + assert "def _is_inverse_conversion" not in _source(_UTILS_PATH), ( + "hydrodatasource/utils/utils.py still defines _is_inverse_conversion, " + "which is only referenced by the dead _validate_inverse_consistency helper." + ) + + +def test_utils__validate_inverse_consistency_removed(): + """The local dead helper ``_validate_inverse_consistency`` must be gone. + + ``streamflow_unit_conv`` imports its own copy from + ``hydroutils.hydro_units``, so the module-level def has no consumers. + """ + assert "def _validate_inverse_consistency" not in _source(_UTILS_PATH), ( + "hydrodatasource/utils/utils.py still defines the local " + "_validate_inverse_consistency, which is never called; the active " + "streamflow_unit_conv uses hydroutils.hydro_units._validate_inverse_consistency." + ) + + +# --------------------------------------------------------------------------- +# B. hydrodatasource/reader/minio_api.py — whole module +# --------------------------------------------------------------------------- + + +def test_minio_api_module_removed(): + """The reader/minio_api module (no importers anywhere) must be deleted.""" + assert not _MINIO_API_PATH.exists(), ( + "hydrodatasource/reader/minio_api.py still exists, but no file in the " + "repository imports it." + ) + + +# --------------------------------------------------------------------------- +# C. hydrodatasource/cleaner/rsvr_inflow_cleaner.py +# --------------------------------------------------------------------------- + + +def test_rsvr_inflow_cleaner__rsvr_rolling_window_abrupt_abnormal_rm_removed(): + """The never-called rolling-window method must be gone.""" + assert "def _rsvr_rolling_window_abrupt_abnormal_rm" not in _source( + _RSVR_CLEANER_PATH + ), ( + "hydrodatasource/cleaner/rsvr_inflow_cleaner.py still defines " + "_rsvr_rolling_window_abrupt_abnormal_rm, which is never called." + ) + + +# --------------------------------------------------------------------------- +# D. hydrodatasource/processor/basin_mean_rainfall.py — voronoi helpers +# --------------------------------------------------------------------------- + + +def test_basin_mean_rainfall_calculate_voronoi_polygons_removed(): + """The deprecated ``calculate_voronoi_polygons`` must be gone.""" + assert "def calculate_voronoi_polygons" not in _source(_BASIN_MEAN_PATH), ( + "hydrodatasource/processor/basin_mean_rainfall.py still defines " + "calculate_voronoi_polygons, which has no callers anywhere in the " + "repository." + ) + + +def test_basin_mean_rainfall_plot_voronoi_polygons_removed(): + """The deprecated ``plot_voronoi_polygons`` must be gone.""" + assert "def plot_voronoi_polygons" not in _source(_BASIN_MEAN_PATH), ( + "hydrodatasource/processor/basin_mean_rainfall.py still defines " + "plot_voronoi_polygons, which has no callers anywhere in the repository." + ) + + +def test_basin_mean_rainfall__plot_voronoi_polygons_removed(): + """The dead private ``_plot_voronoi_polygons`` must be gone.""" + assert "def _plot_voronoi_polygons" not in _source(_BASIN_MEAN_PATH), ( + "hydrodatasource/processor/basin_mean_rainfall.py still defines " + "_plot_voronoi_polygons, which is only referenced by the dead " + "plot_voronoi_polygons wrapper." + ) diff --git a/tests/test_fix_error_handling.py b/tests/test_fix_error_handling.py new file mode 100644 index 0000000..c89b171 --- /dev/null +++ b/tests/test_fix_error_handling.py @@ -0,0 +1,164 @@ +"""Regression tests for three silent error-masking bugs. + +Bug A — hydrodatasource/cleaner/streamflow_cleaner.py:318-323 + ``adaptive_moving_average`` wraps the window slice + ``streamflow_data[start_date:end_date]`` in ``try/except KeyError`` whose + handler only does ``print("WTF")``. When the slice raises a KeyError the + helper variable ``window_data`` is never bound, so the very next line + ``window_data.mean()`` dies with ``UnboundLocalError`` (a ``NameError`` + subclass) — the real KeyError is swallowed and masked. Debug output + ("WTF") also leaks to stdout. + +Bug B — hydrodatasource/utils/utils.py:432-453 + ``minio_file_list`` wraps ``FS.ls()`` in ``except Exception`` that prints + and returns ``[]``, silently swallowing every S3 failure. Downstream code + (reader/data_source.py:438) calls ``minio_file_list(ts_dir)[0]``, so any + transient S3 error surfaces as a confusing ``IndexError: list index out + of range`` instead of the real cause. + +Bug C — hydrodatasource/utils/utils.py:456-482 + ``is_minio_folder``: + * on an empty-but-existing directory, ``objects[0]`` raises IndexError + which is caught and re-raised as NotImplementedError — an empty folder + should return True; and + * rebuilds the candidate path with ``test_object = "s3://" + objects[0]`` + even when the ``ls()`` result already carries the ``s3://`` prefix, so a + single-file path is misreported as a folder. + +These tests reproduce the bugs (RED on the current code) and act as the +regression guard for the fixes. +""" + + +def _keyerror_on_slice_series(data, index): + """Build a pd.Series whose slice access raises KeyError. + + pandas >= 2.2 clamps label slices on a unique DatetimeIndex instead of + raising, so a natural input cannot exercise the ``except KeyError`` branch + in ``adaptive_moving_average``. This test double raises KeyError on *any* + slice access — the exact failure the original ``except KeyError`` was + written to guard against. + """ + import pandas as pd + + class _KeyErrorOnSliceSeries(pd.Series): + def __getitem__(self, key): + if isinstance(key, slice): + raise KeyError("simulated missing time label in slice") + return super().__getitem__(key) + + return _KeyErrorOnSliceSeries(data, index=index) + + +# ── Bug A: adaptive_moving_average must not mask KeyError as NameError ───── + + +def test_adaptive_moving_average_surfaces_keyerror_not_nameerror_when_slice_fails(): + """A failing time-label slice must surface the real KeyError. + + The current code catches the KeyError, then hits ``window_data.mean()`` on + the unbound ``window_data`` and raises ``UnboundLocalError`` (a NameError + subclass). The expected behaviour is that the KeyError propagates. + """ + import numpy as np + import pandas as pd + import pytest + + from hydrodatasource.cleaner.streamflow_cleaner import StreamflowCleaner + + index = pd.date_range("2020-01-01", periods=10, freq="h") + series = _keyerror_on_slice_series(np.arange(10.0), index=index) + cleaner = StreamflowCleaner.__new__(StreamflowCleaner) + + with pytest.raises(KeyError): + cleaner.adaptive_moving_average(series) + + +def test_adaptive_moving_average_does_not_print_wtf_on_missing_slice(capsys): + """The missing-slice path must not emit the "WTF" debug print.""" + import numpy as np + import pandas as pd + import pytest + + from hydrodatasource.cleaner.streamflow_cleaner import StreamflowCleaner + + index = pd.date_range("2020-01-01", periods=10, freq="h") + series = _keyerror_on_slice_series(np.arange(10.0), index=index) + cleaner = StreamflowCleaner.__new__(StreamflowCleaner) + + with pytest.raises(Exception): + cleaner.adaptive_moving_average(series) + + captured = capsys.readouterr() + assert "WTF" not in captured.out + + +# ── Bug B: minio_file_list must not silently swallow FS.ls() errors ──────── + + +def test_minio_file_list_propagates_ls_error_instead_of_returning_empty_list( + monkeypatch, +): + """An exception from ``FS.ls()`` must propagate, not become ``[]``. + + The current code catches every exception, prints it, and returns ``[]`` — + which turns an S3 hiccup into a misleading ``IndexError`` at the call site + (data_source.py:438 does ``minio_file_list(ts_dir)[0]``). + """ + import pytest + + import hydrodatasource.utils.utils as utils + + class _FailingFs: + def ls(self, url): + raise RuntimeError("simulated s3 listing error") + + monkeypatch.setattr(utils, "FS", _FailingFs()) + + with pytest.raises(RuntimeError): + utils.minio_file_list("s3://bucket/dir") + + +# ── Bug C: is_minio_folder edge-case classification ──────────────────────── + + +def test_is_minio_folder_empty_existing_directory_returns_true(monkeypatch): + """An existing but empty directory (no trailing slash) is still a folder. + + The current code indexes ``objects[0]`` on the empty ``ls()`` result -> + IndexError -> swallowed into NotImplementedError, instead of True. + """ + import hydrodatasource.utils.utils as utils + + class _EmptyDirFs: + def exists(self, url): + return True + + def ls(self, url): + return [] + + monkeypatch.setattr(utils, "FS", _EmptyDirFs()) + + assert utils.is_minio_folder("s3://bucket/emptydir") is True + + +def test_is_minio_folder_single_file_path_returns_false(monkeypatch): + """A path that resolves to a single file must not be reported as a folder. + + The current code double-prefixes the first object + (``test_object = "s3://" + objects[0]``) even though the ``ls()`` result + already carries the ``s3://`` prefix, so the single-file path is + misreported as a folder (returns True) instead of False. + """ + import hydrodatasource.utils.utils as utils + + class _SingleFileFs: + def exists(self, url): + return True + + def ls(self, url): + return ["s3://bucket/dir/file.csv"] + + monkeypatch.setattr(utils, "FS", _SingleFileFs()) + + assert utils.is_minio_folder("s3://bucket/dir/file.csv") is False diff --git a/tests/test_fix_grdc_ordering.py b/tests/test_fix_grdc_ordering.py new file mode 100644 index 0000000..8a7570d --- /dev/null +++ b/tests/test_fix_grdc_ordering.py @@ -0,0 +1,115 @@ +"""RED regression tests for the GRDC grdc_no ordering check and is_minio_folder's ls() path. + +A — hydrodatasource/reader/grdc.py:88 ``read_site_info`` validates that the + shapefile's ``grdc_no`` column is ascending with + ``all(x < y for x, y in zip(...))`` over the *string* forms of the IDs. + That is a lexicographic comparison, not a numeric one, so a + variable-length ID pair like ``["999", "1000"]`` (numerically ascending + but lexicographically descending because '9' > '1') is wrongly rejected, + while ``["1000", "999"]`` (numerically descending) is wrongly accepted. + Expected: compare the IDs as numbers. + +B — hydrodatasource/utils/utils.py ``is_minio_folder`` must propagate an + ``FS.ls()`` failure verbatim even when ``FS.exists()`` already returned + True. The strict ``type(...) is RuntimeError`` assertion pins the exact + type so a NotImplementedError wrap (a RuntimeError subclass) cannot + silently satisfy ``pytest.raises(RuntimeError)``. + +These tests reproduce the ordering bug (RED on the current code) and act as +the regression guard for the implementer's fix. Imports live inside the test +functions so a broken module surfaces as a per-test failure rather than a +collection error. +""" + + +def _make_grdc_site_reader(grdc_no_list, monkeypatch): + """Return a Grdc instance ready for ``read_site_info``. + + ``gpd.read_file`` is stubbed to return a DataFrame whose ``grdc_no`` + column is ``grdc_no_list``. ``Grdc.__new__(Grdc)`` skips ``__init__``; + ``read_site_info`` only needs ``data_source_description``. + """ + from types import SimpleNamespace + + import pandas as pd + + from hydrodatasource.reader import grdc as grdc_module + from hydrodatasource.reader.grdc import Grdc + + fake_df = pd.DataFrame( + { + "grdc_no": grdc_no_list, + "area": [float(i) for i in range(len(grdc_no_list))], + } + ) + fake_gpd = SimpleNamespace(read_file=lambda path: fake_df) + monkeypatch.setattr(grdc_module, "gpd", fake_gpd) + + obj = Grdc.__new__(Grdc) + obj.data_source_description = {"BASINS_SHP_FILE": "fake.shp"} + return obj + + +# ── A. read_site_info must validate grdc_no ordering numerically ──────────── + + +def test_grdc_read_site_info_accepts_numeric_ordered_variable_length_ids(monkeypatch): + """['999', '1000'] is numerically ascending and must not raise. + + The current lexicographic comparison sees '999' > '1000' and wrongly + raises ValueError. Expected: numeric comparison accepts the pair. + """ + _make_grdc_site_reader(["999", "1000"], monkeypatch).read_site_info() + + +def test_grdc_read_site_info_rejects_numeric_unsorted_variable_length_ids(monkeypatch): + """['1000', '999'] is numerically descending and must raise ValueError. + + The current lexicographic comparison sees '1000' < '999' and wrongly + accepts the unsorted pair. Expected: numeric comparison rejects it. + """ + import pytest + + with pytest.raises(ValueError): + _make_grdc_site_reader(["1000", "999"], monkeypatch).read_site_info() + + +def test_grdc_read_site_info_accepts_zero_padded_ordered_ids(monkeypatch): + """['0005', '0010'] is numerically ascending and must not raise (regression). + + Fixed-width zero-padded IDs are already accepted by the lexicographic + check; the numeric comparison must keep accepting them. + """ + _make_grdc_site_reader(["0005", "0010"], monkeypatch).read_site_info() + + +# ── B. is_minio_folder must propagate FS.ls() errors verbatim ─────────────── + + +def test_is_minio_folder_propagates_ls_error_when_exists_true(monkeypatch): + """An FS.ls() failure after FS.exists()==True must propagate the original + RuntimeError. + + The strict ``type(...) is RuntimeError`` pins the exact exception type so + a NotImplementedError wrap (a RuntimeError subclass) cannot silently pass + ``pytest.raises(RuntimeError)``. + """ + import pytest + + import hydrodatasource.utils.utils as utils + + class _LsRaisingFs: + def exists(self, url): + return True + + def ls(self, url): + raise RuntimeError("simulated s3 listing error") + + monkeypatch.setattr(utils, "FS", _LsRaisingFs()) + + with pytest.raises(RuntimeError) as excinfo: + utils.is_minio_folder("s3://bucket/dir") + assert type(excinfo.value) is RuntimeError, ( + f"FS.ls() failure must propagate the original RuntimeError, got " + f"{type(excinfo.value).__name__}" + ) diff --git a/tests/test_fix_nits.py b/tests/test_fix_nits.py new file mode 100644 index 0000000..8512f83 --- /dev/null +++ b/tests/test_fix_nits.py @@ -0,0 +1,115 @@ +"""Regression tests for four nits identified in the review cycle. + +Each test is written to FAIL (RED) against the current code and to PASS once +the corresponding fix lands (GREEN): + +- A: ``RainfallAnalyzer.__init__`` must not expose the dead ``logger_level`` + parameter (nor store it as an instance attribute). +- B: ``HydroData(uri="")`` must raise ``ValueError``, not a cryptic + ``FileNotFoundError`` from a bogus relative path. +- C: ``rainfall_runoff_event_identify`` / ``get_rr_events`` in + ``dmca_esr.py`` must not print debug output to stdout. +- D: ``scripts/station_dataset_usage.py`` must not pass the legacy + ``data_path=`` / ``dataset_name=`` keyword arguments (replaced by ``uri=``). + +Only this test file is created; no source files are modified. +""" + +import pytest + + +# ── A. RainfallAnalyzer.logger_level is a dead parameter ── + + +class TestRainfallAnalyzerLoggerLevel: + """``logger_level`` on ``RainfallAnalyzer`` is never read anywhere.""" + + def test_init_signature_has_no_logger_level_parameter(self): + """RainfallAnalyzer.__init__ must not accept a logger_level argument.""" + import inspect + + from hydrodatasource.cleaner.rainfall_cleaner import RainfallAnalyzer + + parameters = inspect.signature(RainfallAnalyzer.__init__).parameters + assert "logger_level" not in parameters + + def test_instance_has_no_logger_level_attribute(self, tmp_path): + """A RainfallAnalyzer instance must not carry a logger_level attribute.""" + from hydrodatasource.cleaner.rainfall_cleaner import RainfallAnalyzer + + data_dir = tmp_path / "data" + out_dir = tmp_path / "out" + data_dir.mkdir(parents=True) + out_dir.mkdir(parents=True) + + analyzer = RainfallAnalyzer( + data_folder=str(data_dir), + output_folder=str(out_dir), + ) + + assert not hasattr(analyzer, "logger_level") + + +# ── B. HydroData(uri="") must raise ValueError ── + + +class TestHydroDataRejectsEmptyUri: + """An empty-string ``uri`` must be rejected just like ``None``.""" + + def test_empty_string_uri_raises_valueerror(self): + """SelfMadeHydroDataset(uri="") must raise ValueError.""" + from hydrodatasource.reader.data_source import SelfMadeHydroDataset + + with pytest.raises(ValueError): + SelfMadeHydroDataset(uri="") + + +# ── C. dmca_esr.py must not print debug output to stdout ── + + +class TestDmcaEsrNoStdoutPrints: + """The DMCA-ESR event identification functions must stay silent on stdout.""" + + def test_rainfall_runoff_event_identify_has_no_print(self): + """rainfall_runoff_event_identify body must contain no print() call.""" + import inspect + + from hydrodatasource.processor.dmca_esr import rainfall_runoff_event_identify + + source = inspect.getsource(rainfall_runoff_event_identify) + assert "print(" not in source + + def test_get_rr_events_has_no_print(self): + """get_rr_events body must contain no print() call.""" + import inspect + + from hydrodatasource.processor.dmca_esr import get_rr_events + + source = inspect.getsource(get_rr_events) + assert "print(" not in source + + +# ── D. scripts/station_dataset_usage.py must drop legacy kwargs ── + + +class TestStationDatasetUsageNoLegacyKwargs: + """The usage script must construct StationHydroDataset with ``uri=``.""" + + @staticmethod + def _script_text(): + from pathlib import Path + + script_path = ( + Path(__file__).resolve().parent.parent + / "scripts" + / "station_dataset_usage.py" + ) + return script_path.read_text(encoding="utf-8") + + def test_no_legacy_data_path_keyword(self): + """The script must not pass the legacy data_path= argument.""" + assert "data_path=" not in self._script_text() + + def test_no_legacy_dataset_name_keyword(self): + """The script must not pass the legacy dataset_name= argument.""" + assert "dataset_name=" not in self._script_text() diff --git a/tests/test_fix_pint_imports.py b/tests/test_fix_pint_imports.py new file mode 100644 index 0000000..c7dfffa --- /dev/null +++ b/tests/test_fix_pint_imports.py @@ -0,0 +1,121 @@ +"""Regression tests: reader modules must not import pint_xarray as a side effect. + +Root cause +---------- +`hydrodatasource/reader/data_source.py` (lines ~510, ~2786) and +`hydrodatasource/reader/gages.py` (line ~657) contained dead, function-level +`import pint_xarray # noqa: F401` statements that served no purpose in the +function bodies. + +pint-xarray performs a *global* side effect on import: it registers the +``.pint`` xarray accessor on every xarray object (pint_xarray/accessors.py, +module level). Under a full-suite run, tests that execute those functions +first (e.g. ``test_tg_hydro_datasource.py``) caused ``pint_xarray`` to be +imported, which made hydroutils 0.2.0's ``_convert_xarray`` take the +``hasattr(data[key], "pint")`` branch and fail on ``mm/3h`` units with +"ValueError: Unit expression cannot have a scaling factor." + +The fix is to delete those dead import lines. These tests guard the invariant +"importing our reader modules and running the attribute-caching entry points +must not import pint_xarray (and thereby register the global .pint accessor)". + +These tests deliberately avoid marking ``internal_data`` and keep every package +import inside a function body so collection is side-effect free. +""" + +import re +import subprocess +import sys +from pathlib import Path + +# Paths are resolved at import time from this file's location; this does not +# import the package, so it is collection-safe. +_REPO_ROOT = Path(__file__).resolve().parents[1] +_DATA_SOURCE_PATH = _REPO_ROOT / "hydrodatasource" / "reader" / "data_source.py" +_GAGES_PATH = _REPO_ROOT / "hydrodatasource" / "reader" / "gages.py" + +# Matches a line that imports pint_xarray (either form), ignoring indentation. +_IMPORT_RE = re.compile( + r"^\s*(?:import\s+pint_xarray\b|from\s+pint_xarray\b)", re.MULTILINE +) + +# Run in a fresh subprocess so sys.modules starts clean and the global side +# effect of importing pint_xarray is observable. +_SUBPROCESS_SNIPPET = r""" +import sys + +import hydrodatasource.reader.data_source as ds +import hydrodatasource.reader.gages as gages + +# Baseline: merely importing the reader modules must not pull pint_xarray in. +if "pint_xarray" in sys.modules: + print("BASELINE: pint_xarray already imported by a reader module") + sys.exit(1) + + +def _run(entry_point, cls): + # Call an attribute-caching entry point on a bare instance so it executes + # through its first statements (where the dead `import pint_xarray` lived), + # then swallow the AttributeError raised because __init__ setup is absent. + try: + entry_point.__get__(object.__new__(cls))() + except Exception: + pass + + +_run(ds.SelfMadeHydroDataset.cache_attributes_xrdataset, ds.SelfMadeHydroDataset) +_run(ds.TgHydroDatasource.cache_intermediate_attributes_xrdataset, ds.TgHydroDatasource) +_run(gages.Gages.cache_attributes_xrdataset, gages.Gages) + +if "pint_xarray" in sys.modules: + print("SIDE EFFECT: pint_xarray imported by attribute-caching entry point") + sys.exit(2) + +print("CLEAN") +sys.exit(0) +""" + + +def test_data_source_has_no_pint_xarray_import(): + """The data_source reader must not contain any pint_xarray import statement.""" + source = _DATA_SOURCE_PATH.read_text(encoding="utf-8") + assert not _IMPORT_RE.search(source), ( + "hydrodatasource/reader/data_source.py still contains an " + "`import pint_xarray` statement; importing it registers the global " + ".pint xarray accessor and breaks hydroutils unit conversion under a " + "full-suite run." + ) + + +def test_gages_has_no_pint_xarray_import(): + """The gages reader must not contain any pint_xarray import statement.""" + source = _GAGES_PATH.read_text(encoding="utf-8") + assert not _IMPORT_RE.search(source), ( + "hydrodatasource/reader/gages.py still contains an " + "`import pint_xarray` statement; importing it registers the global " + ".pint xarray accessor and breaks hydroutils unit conversion under a " + "full-suite run." + ) + + +def test_reader_attribute_caching_does_not_import_pint_xarray_side_effect(): + """Running the attribute-caching entry points must not import pint_xarray. + + In a fresh subprocess: import the reader modules, then execute the three + attribute-caching methods that historically contained a dead + ``import pint_xarray``. If any of them imports pint_xarray, the process + exits non-zero (returncode 2), failing this test. + """ + proc = subprocess.run( + [sys.executable, "-c", _SUBPROCESS_SNIPPET], + cwd=str(_REPO_ROOT), + capture_output=True, + text=True, + timeout=180, + ) + assert proc.returncode == 0, ( + "Importing hydrodatasource.reader.data_source / gages and running the " + "attribute-caching entry points imported pint_xarray as a global side " + "effect (which registers the .pint xarray accessor process-wide). " + f"stdout={proc.stdout!r} stderr={proc.stderr!r}" + ) diff --git a/tests/test_fix_suggestions.py b/tests/test_fix_suggestions.py new file mode 100644 index 0000000..59422cd --- /dev/null +++ b/tests/test_fix_suggestions.py @@ -0,0 +1,221 @@ +"""Regression tests for four Suggestion-level review findings. + +A — hydrodatasource/utils/utils.py ``is_minio_folder`` + wraps *every* exception in ``except Exception`` into NotImplementedError, + including the FileNotFoundError it raises itself (missing path) and any + underlying FS access failure. Expected: real errors propagate unchanged. + +B — hydrodatasource/cleaner/streamflow_cleaner.py:443 + ``self.processed_df[methods[0]][self.origin_df["INQ"].isna()] = np.nan`` + is a chained assignment that fires pandas SettingWithCopyWarning (and the + pandas 3.0 ``ChainedAssignmentError`` FutureWarning). Expected: use + ``.loc`` so no such warning is emitted. + +C — hydrodatasource/reader/grdc.py + 1. the ``if __name__ == "__main__":`` block calls ``cache_grdc_daily()``, + so simply running ``python grdc.py`` kicks off a full cache build. + Expected: the block no longer triggers a full build. + 2. ``read_site_info()`` validates grdc_no ordering with ``assert``, which + is stripped under ``python -O``. Expected: explicit ``raise ValueError``. + +D — tests/test_hydroutils_compat.py + ``assert len(hydroutils_deps) == 1`` falsely fails if pyproject ever adds + hydroutils[extra] or a platform marker. Relaxed to ``assert hydroutils_deps`` + (>=1) in a separate change; this file does not repeat the fragile pattern. + +All tests are RED on the current code and are the regression guard for the +implementer's fix. Imports live inside the test functions so a broken module +surfaces as a per-test failure rather than a collection error. +""" + + +# ── A. is_minio_folder must propagate real errors (not NotImplementedError) ── + + +def test_is_minio_folder_propagates_file_not_found_when_path_missing(monkeypatch): + """A missing path (FS.exists() -> False) must raise FileNotFoundError. + + The current code catches the FileNotFoundError it raises itself and + re-wraps it as NotImplementedError. + """ + import pytest + + import hydrodatasource.utils.utils as utils + + class _MissingFs: + def exists(self, url): + return False + + monkeypatch.setattr(utils, "FS", _MissingFs()) + + with pytest.raises(FileNotFoundError): + utils.is_minio_folder("s3://bucket/missing") + + +def test_is_minio_folder_propagates_fs_access_error(monkeypatch): + """An FS access failure must propagate its original exception type. + + The current code wraps the underlying RuntimeError into + NotImplementedError, hiding the real cause from callers. + """ + import pytest + + import hydrodatasource.utils.utils as utils + + class _RaisingFs: + def exists(self, url): + raise RuntimeError("s3 down") + + monkeypatch.setattr(utils, "FS", _RaisingFs()) + + # NotImplementedError is a subclass of RuntimeError, so pytest.raises alone + # would silently accept the current (wrong) NotImplementedError wrap. Pin + # the exact type to make the test genuinely RED on the current code. + with pytest.raises(RuntimeError) as excinfo: + utils.is_minio_folder("s3://bucket/dir") + assert type(excinfo.value) is RuntimeError, ( + f"FS access failure must propagate the original RuntimeError, got " + f"{type(excinfo.value).__name__}" + ) + + +# ── B. anomaly_process must not use chained assignment ─────────────────────── + + +def _write_streamflow_csv(tmp_path): + """Write a minimal streamflow CSV (TM + INQ columns) and return its path.""" + import pandas as pd + + csv_path = tmp_path / "streamflow_input.csv" + df = pd.DataFrame( + { + "TM": pd.date_range("2023-01-01", periods=48, freq="D"), + "INQ": [100, 110, 120, 130, 95, 105, 115, 125] * 6, + } + ) + df.to_csv(csv_path, index=False) + return csv_path + + +def test_anomaly_process_does_not_emit_setting_with_copy_warning(tmp_path): + """anomaly_process must complete without a pandas SettingWithCopyWarning. + + Line 443 writes through a chained ``df[col][mask] = value``; the fix must + use ``.loc``. The CSV fixture has no NaN in INQ, but the warning fires on + the chained assignment regardless of the mask content (verified on + pandas 2.3.3). + """ + import warnings + + import pandas as pd + + from hydrodatasource.cleaner.streamflow_cleaner import StreamflowCleaner + + csv_path = _write_streamflow_csv(tmp_path) + cleaner = StreamflowCleaner(data_folder=str(csv_path)) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + cleaner.anomaly_process(["moving_average"]) + + masked_warnings = [ + w + for w in caught + if issubclass(w.category, pd.errors.SettingWithCopyWarning) + ] + assert masked_warnings == [], ( + "anomaly_process emitted SettingWithCopyWarning (chained assignment " + "in the 'remove pre-interpolated NaNs' step): " + f"{[str(w.message) for w in masked_warnings]}" + ) + + +def test_anomaly_process_does_not_use_chained_assignment_future_warning(tmp_path): + """anomaly_process must not rely on chained assignment that pandas 3.0 breaks. + + pandas 2.3 emits a FutureWarning ("ChainedAssignmentError: behaviour will + change in pandas 3.0!") for the same chained assignment on line 443. + """ + import warnings + + from hydrodatasource.cleaner.streamflow_cleaner import StreamflowCleaner + + csv_path = _write_streamflow_csv(tmp_path) + cleaner = StreamflowCleaner(data_folder=str(csv_path)) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + cleaner.anomaly_process(["EMA"]) + + chained_assignment = [ + w + for w in caught + if issubclass(w.category, FutureWarning) + and "chained assignment" in str(w.message).lower() + ] + assert chained_assignment == [], ( + "anomaly_process relies on chained assignment: " + f"{[str(w.message) for w in chained_assignment]}" + ) + + +# ── C1. grdc.py __main__ block must not trigger a full cache build ─────────── + + +def test_grdc_main_block_does_not_trigger_full_cache_build(): + """Running ``python hydrodatasource/reader/grdc.py`` must not build the cache. + + The current ``if __name__ == "__main__":`` block calls + ``grdc.cache_grdc_daily()`` with no station filter, which triggers a full + GRDC cache build on direct execution. The block must be removed or guarded + so that only explicitly-invoked code builds the cache. + """ + from pathlib import Path + + src_path = ( + Path(__file__).resolve().parents[1] / "hydrodatasource" / "reader" / "grdc.py" + ) + text = src_path.read_text(encoding="utf-8") + + main_marker = 'if __name__ == "__main__":' + main_index = text.find(main_marker) + if main_index == -1: + # No __main__ block at all — nothing to guard against. + return + + main_block = text[main_index:] + assert "cache_grdc_daily" not in main_block, ( + "the __main__ block must not call cache_grdc_daily() (full cache build)" + ) + + +# ── C2. read_site_info must validate with ValueError, not assert ───────────── + + +def test_grdc_read_site_info_raises_value_error_on_unsorted_ids(monkeypatch): + """Unordered grdc_no must raise ValueError, not an assert (stripped under -O). + + ``read_site_info`` currently uses ``assert all(x < y ...)`` to validate + that the shapefile's grdc_no is ascending — that check silently vanishes + under ``python -O``. The expected behaviour is an explicit + ``raise ValueError``. + """ + from types import SimpleNamespace + + import pandas as pd + import pytest + + from hydrodatasource.reader import grdc as grdc_module + from hydrodatasource.reader.grdc import Grdc + + fake_df = pd.DataFrame( + {"grdc_no": [5.0, 3.0, 7.0], "area": [1.0, 2.0, 3.0]} + ) + fake_gpd = SimpleNamespace(read_file=lambda path: fake_df) + monkeypatch.setattr(grdc_module, "gpd", fake_gpd) + + obj = Grdc.__new__(Grdc) + obj.data_source_description = {"BASINS_SHP_FILE": "fake.shp"} + + with pytest.raises(ValueError): + obj.read_site_info() diff --git a/tests/test_fix_unit_mapping.py b/tests/test_fix_unit_mapping.py new file mode 100644 index 0000000..b070645 --- /dev/null +++ b/tests/test_fix_unit_mapping.py @@ -0,0 +1,119 @@ +"""Regression tests for the ``streamflow_unit`` property of ``SelfMadeHydroDataset``. + +Two defects (from review of ``hydrodatasource/reader/data_source.py``): + +- The ``time_unit`` whitelist accepts ``"8D"`` (constructor check) but the + ``streamflow_unit`` ``unit_mapping`` only covers ``{"1h", "3h", "1D", "1M"}``, + so ``SelfMadeHydroDataset(time_unit=["8D"]).streamflow_unit`` raises + ``KeyError: '8D'``. +- ``time_unit=None`` makes ``streamflow_unit`` iterate over ``None`` and raise + ``TypeError: 'NoneType' object is not iterable`` instead of returning an + empty mapping (the tolerant style used elsewhere, e.g. ``or ["1D"]`` in + ``read_ts_xrdataset``). + +Each test FAILS (RED) against the current implementation and PASSES once the +property maps ``"8D"`` to ``"mm/8d"`` (the canonical string used elsewhere in +the module, e.g. in the precipitation-unit conversion) and treats +``time_unit=None`` as an empty mapping. + +The tests build a minimal Caravan-style dataset directory under ``tmp_path`` +so the constructor's directory inspection works; they never touch real data. +""" + + +def _make_dataset_dir(tmp_path): + """Build a minimal Caravan-style dataset directory. + + Structure:: + + /selfmade_ds/ + ├── attributes/ + │ └── attributes.csv # basin_id,area + └── timeseries/ + ├── 1D/ # empty time-unit dir (for _where_ts_dir) + └── 1M/ # empty time-unit dir (for _where_ts_dir) + + The ``timeseries`` subdirectories are required because + ``set_data_source_describe`` lists them via ``_where_ts_dir``. + """ + data_dir = tmp_path / "selfmade_ds" + + attrs_dir = data_dir / "attributes" + attrs_dir.mkdir(parents=True) + (attrs_dir / "attributes.csv").write_text( + "basin_id,area\n01015500,12345\n01020500,67890\n", + encoding="utf-8", + ) + + ts_dir = data_dir / "timeseries" + (ts_dir / "1D").mkdir(parents=True) + (ts_dir / "1M").mkdir(parents=True) + + return data_dir + + +class TestStreamflowUnit8D: + """``streamflow_unit`` must map the whitelisted ``"8D"`` time unit.""" + + def test_streamflow_unit_maps_8d(self, tmp_path): + """streamflow_unit maps "8D" -> "mm/8d" (canonical unit string).""" + from hydrodatasource.reader.data_source import SelfMadeHydroDataset + + ds = SelfMadeHydroDataset( + uri=str(_make_dataset_dir(tmp_path)), + time_unit=["8D"], + ) + + assert ds.streamflow_unit == {"8D": "mm/8d"} + + def test_streamflow_unit_covers_all_supported_units_including_8d(self, tmp_path): + """streamflow_unit covers the full supported set including "8D".""" + from hydrodatasource.reader.data_source import SelfMadeHydroDataset + + ds = SelfMadeHydroDataset( + uri=str(_make_dataset_dir(tmp_path)), + time_unit=["1h", "3h", "1D", "8D", "1M"], + ) + + assert ds.streamflow_unit == { + "1h": "mm/h", + "3h": "mm/3h", + "1D": "mm/d", + "8D": "mm/8d", + "1M": "mm/M", + } + + +class TestStreamflowUnitNone: + """``streamflow_unit`` must tolerate ``time_unit=None`` by returning {}.""" + + def test_streamflow_unit_empty_when_time_unit_none(self, tmp_path): + """time_unit=None yields an empty mapping instead of raising TypeError.""" + from hydrodatasource.reader.data_source import SelfMadeHydroDataset + + ds = SelfMadeHydroDataset( + uri=str(_make_dataset_dir(tmp_path)), + time_unit=None, + ) + + assert ds.streamflow_unit == {} + + +class TestStreamflowUnitNoRegression: + """Existing legal units keep their canonical mappings (no regression).""" + + def test_streamflow_unit_maps_legal_units(self, tmp_path): + """1h/3h/1D/1M keep their existing mappings.""" + from hydrodatasource.reader.data_source import SelfMadeHydroDataset + + ds = SelfMadeHydroDataset( + uri=str(_make_dataset_dir(tmp_path)), + time_unit=["1h", "3h", "1D", "1M"], + ) + + assert ds.streamflow_unit == { + "1h": "mm/h", + "3h": "mm/3h", + "1D": "mm/d", + "1M": "mm/M", + } diff --git a/tests/test_hydroutils_compat.py b/tests/test_hydroutils_compat.py index ed51243..d647a8d 100644 --- a/tests/test_hydroutils_compat.py +++ b/tests/test_hydroutils_compat.py @@ -100,8 +100,8 @@ def test_pyproject_requires_hydroutils_0_2_or_newer(): hydroutils_deps = [ d for d in deps if d.strip().lower().startswith("hydroutils") ] - assert len(hydroutils_deps) == 1, ( - "expected exactly one hydroutils entry in [project].dependencies" + assert hydroutils_deps, ( + "expected at least one hydroutils entry in [project].dependencies" ) req = Requirement(hydroutils_deps[0]) diff --git a/uv.lock b/uv.lock index d16c669..cc50955 100644 --- a/uv.lock +++ b/uv.lock @@ -267,15 +267,15 @@ wheels = [ [[package]] name = "aqua-fetch" -version = "1.0.1" +version = "1.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pandas" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a9/7e/b4343a680a1f0b2356f6cf9371dded8141c0d8ea61ec7b41c6cdbedab3fe/aqua_fetch-1.0.1.tar.gz", hash = "sha256:7991a7fa2ab6592ff23dd0f8b9c2e2280b7aca20b01e4731bc3b6f974ded9578", size = 274826, upload-time = "2026-02-11T08:02:55.734Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/17/9f5669133e37314f8f6613f39bc02d03729f1faeb02c116f59e2aec893d1/aqua_fetch-1.1.0.tar.gz", hash = "sha256:bf4eb8dfeaba8b520640bb316a55877a5caf53dc887f01fd7a97704c551e1ceb", size = 352261, upload-time = "2026-07-29T14:53:04.314Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/79/1d/f3803f12c8827cc1846eb79a2e2ddccf164d90833b9d56c829dc14b9f3e8/aqua_fetch-1.0.1-py3-none-any.whl", hash = "sha256:053ce30fcf9c76564a83dba1ded39e254f629a2d1cc08b3989093b7612a5a6aa", size = 289114, upload-time = "2026-02-11T08:02:54.233Z" }, + { url = "https://files.pythonhosted.org/packages/7c/1a/e8efe904da305ff7ca403c100a6fcaee9a417084b9f3c9e11d67ab27a4d4/aqua_fetch-1.1.0-py3-none-any.whl", hash = "sha256:e1f81b80db541591fdbcd0608cc48da9957053a9d4f07116662edb0decdbaf1d", size = 365775, upload-time = "2026-07-29T14:53:02.538Z" }, ] [package.optional-dependencies] @@ -2342,7 +2342,7 @@ dependencies = [ [package.metadata] requires-dist = [ - { name = "aqua-fetch", extras = ["all"], specifier = ">=1.0.1" }, + { name = "aqua-fetch", extras = ["all"], specifier = ">=1.1.0" }, { name = "black", marker = "extra == 'dev'" }, { name = "black", marker = "extra == 'lint'" }, { name = "bokeh", specifier = ">=2.4.2" }, From b9988811aa498b17526095a2266db3183ff7c2bc Mon Sep 17 00:00:00 2001 From: owen Date: Sat, 15 Aug 2026 19:21:20 +0800 Subject: [PATCH 29/34] docs: revise docs for unified URI-only interface and pin hydrodataset>=0.3.0 README (en/zh): - describe the unified URI-only design and hydrodataset compatibility (drop internal ADR framing) - add open_dataset() quick start; URI-only SelfMadeHydroDataset construction - 11 reader alias table; storage.* config; directory structure with intermediate/stations/8D docs/: - reader: open_dataset/resolve_data_path as the main entry, URI-only examples, full alias table - configs: storage.* unified format + default_source + resolution flow - api: add data_resolver section and all reader classes - cleaner: note the functional Cleaner base and correct algorithm names - remove spec-unified-data-interface.md (design now lives in data_resolver.py docstring) pyproject: hydrodataset>=0.2.0 -> >=0.3.0 (configs.settings and configs.data_resolver were introduced in hydrodataset 0.3; the old floor allowed installing an incompatible 0.2.x) --- README.md | 135 ++++++++++++++------ README.zh.md | 133 ++++++++++++++------ docs/api.md | 104 +++++++++++++++ docs/cleaner.md | 17 ++- docs/configs.md | 64 +++++++--- docs/reader.md | 91 +++++++++++--- docs/spec-unified-data-interface.md | 188 ---------------------------- pyproject.toml | 2 +- 8 files changed, 436 insertions(+), 298 deletions(-) delete mode 100644 docs/spec-unified-data-interface.md diff --git a/README.md b/README.md index e2477c3..738cf72 100644 --- a/README.md +++ b/README.md @@ -11,65 +11,83 @@ While libraries like [hydrodataset](https://github.com/OuyangWenyu/hydrodataset) **`hydrodatasource`** is designed to solve this problem. It provides a flexible framework to read, process, and clean these custom datasets, preparing them for hydrological modeling and analysis. -The core of this framework is the `SelfMadeHydroDataset` class, which allows you to easily access your own data by organizing it into a simple, predefined directory structure. +`hydrodatasource` uses a unified **URI-only** design: every reader is constructed by pointing it at a directory or an `s3://` URI, and a one-line `open_dataset()` factory resolves a dataset id to a ready-to-use reader. The design is compatible with [hydrodataset](https://github.com/OuyangWenyu/hydrodataset). -## Reading Custom Datasets with `SelfMadeHydroDataset` +## Quick Start -This is the primary use case for `hydrodatasource`. If you have your own basin-level time series and attribute data, you can use this class to load it seamlessly. +If you have a **registered dataset** and a `~/hydro_setting.yml` (see [Configuration](#configuration)), resolve and open it in one line: + +```python +from hydrodatasource.configs.data_resolver import open_dataset + +# "songliao_event" is a hydrodatasource-registered dataset (flood-event reader) +ds = open_dataset("songliao_event") +``` + +To **open a custom dataset** by path, construct the reader directly: + +```python +from hydrodatasource.reader.data_source import SelfMadeHydroDataset + +reader = SelfMadeHydroDataset(uri="/path/to/my_dataset", time_unit=["1D"]) +``` + +## Reading Custom Datasets + +This is the primary use case for `hydrodatasource`. If you have your own basin-level time series and attribute data, you can use `SelfMadeHydroDataset` to load it seamlessly. ### 1. Prepare Your Data Directory First, organize your data into the following folder structure: ``` -/path/to/your_data_root/ - └── my_custom_dataset/ # Your dataset's name - ├── attributes/ - │ └── attributes.csv - ├── shapes/ - │ └── basins.shp - └── timeseries/ - ├── 1D/ # Sub-folder for each time resolution (e.g., daily) - │ ├── basin_01.csv - │ ├── basin_02.csv - │ └── ... - └── 1D_units_info.json # JSON file with unit information +/path/to/my_dataset/ +├── attributes/ +│ └── attributes.csv +├── shapes/ +│ └── basins.shp +└── timeseries/ + ├── 1D/ # Sub-folder for each time resolution (e.g., daily) + │ ├── basin_01.csv + │ ├── basin_02.csv + │ └── ... + └── 1D_units_info.json # JSON file with unit information ``` - **`attributes/attributes.csv`**: A CSV file containing static basin attributes (e.g., area, mean elevation). Must include a `basin_id` column that matches the filenames in the `timeseries` folder. - **`shapes/basins.shp`**: A shapefile with the polygon geometry for each basin. -- **`timeseries/1D/`**: A folder for each time resolution (e.g., `1D` for daily, `3h` for 3-hourly). Inside, each CSV file should contain the time series data for a single basin and be named after its `basin_id`. -- **`timeseries/1D_units_info.json`**: A JSON file defining the units for each variable in your time series CSVs (e.g., `{"precipitation": "mm/d", "streamflow": "m3/s"}`). +- **`timeseries/1D/`**: A folder for each time resolution (`1h`, `3h`, `1D`, `8D`, `1M`). Inside, each CSV file should contain the time series data for a single basin and be named after its `basin_id`. +- **`timeseries/1D_units_info.json`**: A JSON file defining the units for each variable in your time series CSVs (e.g., `{"precipitation": "mm/d", "streamflow": "m3/s", "temperature": "degC"}`). Every variable you read must be listed here. + +Extended datasets may add optional directories: + +- **`intermediate/`** — interval-basin (区间流域) data with topological relationships (`attributes/`, `timeseries/`, `shapes/`), used by the TG basin reader. +- **`stations/`** — gauging-station data and adjacency matrices, used by the station reader. +- **`forecasts/`** — forecast time series, used by the forecast reader. ### 2. Read the Data in Python -Once your data is organized, you can use `SelfMadeHydroDataset` to read it with just a few lines of code. +Once your data is organized, point a URI-only reader at it: ```python from hydrodatasource.reader.data_source import SelfMadeHydroDataset -# 1. Define the path to your data's parent directory and the dataset name -data_path = "/path/to/your_data_root/" -dataset_name = "my_custom_dataset" - -# 2. Initialize the reader -# Specify the time units you want to work with -reader = SelfMadeHydroDataset(data_path=data_path, dataset_name=dataset_name, time_unit=["1D"]) +# Pass the absolute path (or s3:// URI) of your dataset directory as `uri` +reader = SelfMadeHydroDataset(uri="/path/to/my_dataset", time_unit=["1D"]) -# 3. Get a list of all available basin IDs +# Get a list of all available basin IDs basin_ids = reader.read_object_ids() -# 4. Define the time range and variables you want to load +# Define the time range and variables you want to load t_range = ["2000-01-01", "2010-12-31"] variables_to_read = ["precipitation", "streamflow", "temperature"] -# 5. Read the time series data -# The result is a dictionary of xarray.Datasets, keyed by time unit +# Read the time series data (a dict of xarray.Datasets keyed by time unit) timeseries_data = reader.read_ts_xrdataset( gage_id_lst=basin_ids, t_range=t_range, var_lst=variables_to_read, - time_units=["1D"] + time_units=["1D"], ) daily_data = timeseries_data["1D"] @@ -77,12 +95,60 @@ daily_data = timeseries_data["1D"] print("Successfully loaded data:") print(daily_data) -# You can also read the static attributes +# Static attributes are equally easy to read attributes_data = reader.read_attr_xrdataset(gage_id_lst=basin_ids, var_lst=["area", "mean_elevation"]) print("\nAttributes:") print(attributes_data) ``` +> **Note on the old API.** Earlier versions accepted `data_path=` / `dataset_name=` constructor arguments. +> These were removed in favor of the unified `uri=` interface and now raise `ValueError`. + +## Reader Aliases + +All `hydrodatasource` readers are registered as aliases in `READER_ALIASES`, which downstream +projects (e.g. hydromodel) consume alongside hydrodataset's aliases: + +| Alias | Class | Directory convention | +|-------|-------|---------------------| +| `selfmade` | `SelfMadeHydroDataset` | standard dataset (`attributes/`, `timeseries/`, `shapes/`) | +| `longterm` | `LongTermDataset` | self-made dataset with long-term support | +| `forecast` | `SelfMadeForecastDataset` | standard + `forecasts/` | +| `station` | `StationHydroDataset` | standard + `stations/` | +| `tghydro` | `TgHydroDatasource` | standard + `intermediate/` + LSTM predictions | +| `floodevent` | `FloodEventDatasource` | flood-event data with per-basin event markers | +| `gages` | `Gages` | GAGES-II public dataset | +| `grdc` | `Grdc` | GRDC public dataset | +| `rainfall` | `RainfallReader` | cleaned station rainfall | +| `crd` | `Crd` | China reservoir database | +| `rsvrinflow` | `RsvrInflowReader` | reservoir inflow data | + +`hydrodataset`'s public datasets (e.g. `camels_us`) are also resolvable through the same +`open_dataset()` / `resolve_data_path()` interface. + +## Configuration + +`hydrodatasource` reads a shared `~/hydro_setting.yml` (same file used by `hydrodataset` and +`hydromodel`) using the unified `storage.*` format: + +```yaml +storage: + default_source: local # 'local' or 'cloud' — fallback when source is not given + local: + root: 'D:\data\hydrodatasource' # main data root + cache: data\cache + s3: # optional — cloud (MinIO/S3) access + endpoint_url: 'http://minio:9000' + key: 'access_key' + secret: 'secret_key' + bucket: hydro-data + prefix: hydromodel +``` + +- When `default_source: local`, `resolve_data_path()` / `open_dataset()` resolve against `storage.local.root`. +- When `default_source: cloud`, they resolve against `storage.s3`. +- If `~/hydro_setting.yml` is missing, a default root `~/hydrodatasource_data` is used (with a warning). + ## Other Features Beyond reading data, `hydrodatasource` also includes modules for: @@ -102,7 +168,7 @@ pip install hydrodatasource ### Development Setup -For developers, it is recommended to use `uv` to manage the environment, as this project has local dependencies (e.g., `hydroutils`, `hydrodataset`). +For developers, it is recommended to use `uv` to manage the environment, as this project has local dependencies (e.g., `hydroutils>=0.2.0`, `hydrodataset>=0.3.0`). 1. **Clone the repository:** ```bash @@ -110,8 +176,7 @@ For developers, it is recommended to use `uv` to manage the environment, as this cd hydrodatasource ``` -2. **Sync the environment with `uv`:** - This command will install all dependencies, including the local editable packages. +2. **Sync the environment with `uv`** (installs all extras, including dev tooling): ```bash uv sync --all-extras - ``` \ No newline at end of file + ``` diff --git a/README.zh.md b/README.zh.md index 653d038..ece331e 100644 --- a/README.zh.md +++ b/README.zh.md @@ -11,77 +11,141 @@ **`hydrodatasource`** 正是为了解决这一问题而设计的。它提供了一个灵活的框架，用于读取、处理和清洗这些自定义数据集，为水文建模和分析做准备。 -该框架的核心是 `SelfMadeHydroDataset` 类，它允许您通过将数据组织成一个简单的、预定义的目录结构来轻松访问您自己的数据。 +`hydrodatasource` 采用统一的 **URI-only** 设计：所有读取器都通过指向一个目录或 `s3://` URI 来构造，一行 `open_dataset()` 工厂即可将数据集 id 解析为可用的读取器。该设计与 [hydrodataset](https://github.com/OuyangWenyu/hydrodataset) 兼容。 -## 使用 `SelfMadeHydroDataset` 读取自定义数据集 +## 快速上手 -这是 `hydrodatasource` 的主要用例。如果您有自己的流域级别时间序列和属性数据，您可以使用此类无缝加载它。 +如果您有**已注册的数据集**和 `~/hydro_setting.yml`（见[配置](#配置)），可一行解析并打开： + +```python +from hydrodatasource.configs.data_resolver import open_dataset + +# "songliao_event" 是 hydrodatasource 注册的数据集（洪水事件读取器） +ds = open_dataset("songliao_event") +``` + +要**按路径打开自定义数据集**，请直接构造读取器： + +```python +from hydrodatasource.reader.data_source import SelfMadeHydroDataset + +reader = SelfMadeHydroDataset(uri="/path/to/my_dataset", time_unit=["1D"]) +``` + +## 读取自定义数据集 + +这是 `hydrodatasource` 的主要用例。如果您有自己的流域级别时间序列和属性数据，您可以使用 `SelfMadeHydroDataset` 无缝加载它。 ### 1. 准备您的数据目录 首先，将您的数据组织成以下文件夹结构： ``` -/path/to/your_data_root/ - └── my_custom_dataset/ # 您的数据集名称 - ├── attributes/ - │ └── attributes.csv - ├── shapes/ - │ └── basins.shp - └── timeseries/ - ├── 1D/ # 每个时间分辨率一个子文件夹（例如，日尺度） - │ ├── basin_01.csv - │ ├── basin_02.csv - │ └── ... - └── 1D_units_info.json # 包含单位信息的 JSON 文件 +/path/to/my_dataset/ +├── attributes/ +│ └── attributes.csv +├── shapes/ +│ └── basins.shp +└── timeseries/ + ├── 1D/ # 每个时间分辨率一个子文件夹（例如，日尺度） + │ ├── basin_01.csv + │ ├── basin_02.csv + │ └── ... + └── 1D_units_info.json # 包含单位信息的 JSON 文件 ``` - **`attributes/attributes.csv`**: 包含静态流域属性（例如，面积、平均高程）的 CSV 文件。必须包含一个 `basin_id` 列，该列与 `timeseries` 文件夹中的文件名匹配。 - **`shapes/basins.shp`**: 包含每个流域多边形几何的 shapefile 文件。 -- **`timeseries/1D/`**: 每个时间分辨率一个文件夹（例如，`1D` 表示日尺度，`3h` 表示 3 小时尺度）。在内部，每个 CSV 文件应包含单个流域的时间序列数据，并以其 `basin_id` 命名。 -- **`timeseries/1D_units_info.json`**: 定义时间序列 CSV 中每个变量单位的 JSON 文件（例如，`{"precipitation": "mm/d", "streamflow": "m3/s"}`）。 +- **`timeseries/1D/`**: 每个时间分辨率一个文件夹（`1h`、`3h`、`1D`、`8D`、`1M`）。在内部，每个 CSV 文件应包含单个流域的时间序列数据，并以其 `basin_id` 命名。 +- **`timeseries/1D_units_info.json`**: 定义时间序列 CSV 中每个变量单位的 JSON 文件（例如，`{"precipitation": "mm/d", "streamflow": "m3/s", "temperature": "degC"}`）。您读取的每个变量都必须在此列出。 + +扩展数据集可选用以下可选目录： + +- **`intermediate/`** — 区间流域数据（含拓扑关系），供 TG 流域读取器使用。 +- **`stations/`** — 测站数据与邻接矩阵，供站点读取器使用。 +- **`forecasts/`** — 预报时间序列，供预报读取器使用。 ### 2. 在 Python 中读取数据 -一旦您的数据被组织好，您就可以使用 `SelfMadeHydroDataset` 通过几行代码来读取它。 +一旦您的数据被组织好，将 URI-only 读取器指向它： ```python from hydrodatasource.reader.data_source import SelfMadeHydroDataset -# 1. 定义数据父目录的路径和数据集名称 -data_path = "/path/to/your_data_root/" -dataset_name = "my_custom_dataset" - -# 2. 初始化读取器 -# 指定您要使用的时间单位 -reader = SelfMadeHydroDataset(data_path=data_path, dataset_name=dataset_name, time_unit=["1D"]) +# 将数据集目录的绝对路径（或 s3:// URI）作为 uri 传入 +reader = SelfMadeHydroDataset(uri="/path/to/my_dataset", time_unit=["1D"]) -# 3. 获取所有可用流域 ID 的列表 +# 获取所有可用流域 ID 的列表 basin_ids = reader.read_object_ids() -# 4. 定义您要加载的时间范围和变量 +# 定义您要加载的时间范围和变量 t_range = ["2000-01-01", "2010-12-31"] variables_to_read = ["precipitation", "streamflow", "temperature"] -# 5. 读取时间序列数据 -# 结果是一个 xarray.Dataset 的字典，以时间单位为键 +# 读取时间序列数据（以时间单位为键的 xarray.Dataset 字典） timeseries_data = reader.read_ts_xrdataset( gage_id_lst=basin_ids, t_range=t_range, var_lst=variables_to_read, - time_units=["1D"]) + time_units=["1D"], +) daily_data = timeseries_data["1D"] print("数据加载成功：") print(daily_data) -# 您也可以读取静态属性 +# 静态属性同样易于读取 attributes_data = reader.read_attr_xrdataset(gage_id_lst=basin_ids, var_lst=["area", "mean_elevation"]) print("\n属性：") print(attributes_data) ``` +> **关于旧 API 的说明。** 早期版本接受 `data_path=` / `dataset_name=` 构造参数。 +> 它们已被统一的 `uri=` 接口取代，现在会抛出 `ValueError`。 + +## 读取器别名 + +所有 `hydrodatasource` 读取器都注册在 `READER_ALIASES` 中，供下游项目（如 hydromodel）与 hydrodataset 的别名一起使用： + +| 别名 | 类 | 目录约定 | +|-------|-------|-----| +| `selfmade` | `SelfMadeHydroDataset` | 标准数据集（`attributes/`、`timeseries/`、`shapes/`） | +| `longterm` | `LongTermDataset` | 支持长期序列的自制数据集 | +| `forecast` | `SelfMadeForecastDataset` | 标准 + `forecasts/` | +| `station` | `StationHydroDataset` | 标准 + `stations/` | +| `tghydro` | `TgHydroDatasource` | 标准 + `intermediate/` + LSTM 预测 | +| `floodevent` | `FloodEventDatasource` | 带逐流域事件标记的洪水事件数据 | +| `gages` | `Gages` | GAGES-II 公共数据集 | +| `grdc` | `Grdc` | GRDC 公共数据集 | +| `rainfall` | `RainfallReader` | 清洗后的站点降雨 | +| `crd` | `Crd` | 中国水库数据库 | +| `rsvrinflow` | `RsvrInflowReader` | 水库入库流量数据 | + +`hydrodataset` 的公共数据集（如 `camels_us`）也可通过同一个 `open_dataset()` / `resolve_data_path()` 接口解析。 + +## 配置 + +`hydrodatasource` 读取共享的 `~/hydro_setting.yml`（与 `hydrodataset`、`hydromodel` 共用），使用统一的 `storage.*` 格式： + +```yaml +storage: + default_source: local # 'local' 或 'cloud' — 未指定 source 时的回退值 + local: + root: 'D:\data\hydrodatasource' # 主数据根目录 + cache: data\cache + s3: # 可选 — 云端（MinIO/S3）访问 + endpoint_url: 'http://minio:9000' + key: 'access_key' + secret: 'secret_key' + bucket: hydro-data + prefix: hydromodel +``` + +- `default_source: local` 时，`resolve_data_path()` / `open_dataset()` 针对 `storage.local.root` 解析。 +- `default_source: cloud` 时，针对 `storage.s3` 解析。 +- 若 `~/hydro_setting.yml` 缺失，将使用默认根目录 `~/hydrodatasource_data`（带警告）。 + ## 其他功能 除了读取数据，`hydrodatasource` 还包括用于以下功能的模块： @@ -101,7 +165,7 @@ pip install hydrodatasource ### 开发设置 -对于开发人员，建议使用 `uv` 来管理环境，因为该项目具有本地依赖项（例如 `hydroutils`、`hydrodataset`）。 +对于开发人员，建议使用 `uv` 来管理环境，因为该项目具有本地依赖项（例如 `hydroutils>=0.2.0`、`hydrodataset>=0.3.0`）。 1. **克隆存储库：** ```bash @@ -109,8 +173,7 @@ pip install hydrodatasource cd hydrodatasource ``` -2. **使用 `uv` 同步环境：** - 此命令将安装所有依赖项，包括本地可编辑的软件包。 +2. **使用 `uv` 同步环境**（安装全部 extras，含 dev 工具）： ```bash uv sync --all-extras - ``` \ No newline at end of file + ``` diff --git a/docs/api.md b/docs/api.md index 2c580d2..91f8556 100644 --- a/docs/api.md +++ b/docs/api.md @@ -2,14 +2,110 @@ This page provides an auto-generated API reference for the key components of the `hydrodatasource` library. +## Data Resolver (Unified Data Interface) + +::: hydrodatasource.configs.data_resolver.open_dataset + handler: python + options: + show_root_heading: true + show_source: false + +::: hydrodatasource.configs.data_resolver.resolve_data_path + handler: python + options: + show_root_heading: true + show_source: false + +::: hydrodatasource.configs.data_resolver.ResolverContext + handler: python + options: + show_root_heading: true + show_source: false + +`READER_ALIASES` maps every reader alias to its module/class, and `HDS_DATASETS` is the in-code +registry of hydrodatasource datasets (e.g. `songliao_event`). `DatasetResolutionError` is +re-exported from hydrodataset. + ## Reader +### HydroData (base class) + +::: hydrodatasource.reader.data_source.HydroData + handler: python + options: + show_root_heading: true + show_source: false + +### SelfMadeHydroDataset + ::: hydrodatasource.reader.data_source.SelfMadeHydroDataset handler: python options: show_root_heading: true show_source: false +### Other Readers + +::: hydrodatasource.reader.data_source.LongTermDataset + handler: python + options: + show_root_heading: true + show_source: false + +::: hydrodatasource.reader.data_source.SelfMadeForecastDataset + handler: python + options: + show_root_heading: true + show_source: false + +::: hydrodatasource.reader.data_source.StationHydroDataset + handler: python + options: + show_root_heading: true + show_source: false + +::: hydrodatasource.reader.data_source.TgHydroDatasource + handler: python + options: + show_root_heading: true + show_source: false + +::: hydrodatasource.reader.floodevent.FloodEventDatasource + handler: python + options: + show_root_heading: true + show_source: false + +::: hydrodatasource.reader.gages.Gages + handler: python + options: + show_root_heading: true + show_source: false + +::: hydrodatasource.reader.grdc.Grdc + handler: python + options: + show_root_heading: true + show_source: false + +::: hydrodatasource.reader.rainfall_reader.RainfallReader + handler: python + options: + show_root_heading: true + show_source: false + +::: hydrodatasource.reader.reservoir_datasets.Crd + handler: python + options: + show_root_heading: true + show_source: false + +::: hydrodatasource.reader.rsvr_inflow_reader.RsvrInflowReader + handler: python + options: + show_root_heading: true + show_source: false + ## Processor ### Basin Mean Rainfall @@ -30,6 +126,14 @@ This page provides an auto-generated API reference for the key components of the ## Cleaner +### Cleaner (base class) + +::: hydrodatasource.cleaner.cleaner.Cleaner + handler: python + options: + show_root_heading: true + show_source: false + ### RainfallCleaner ::: hydrodatasource.cleaner.rainfall_cleaner.RainfallCleaner diff --git a/docs/cleaner.md b/docs/cleaner.md index 0c0fdd2..22b40f0 100644 --- a/docs/cleaner.md +++ b/docs/cleaner.md @@ -2,6 +2,17 @@ The `cleaner` module provides a suite of tools for cleaning and preprocessing raw hydrological time series data. Raw data from gauging stations often contains errors, gaps, and noise. This module helps to identify and correct these issues to prepare the data for analysis and modeling. +## Base `Cleaner` + +All cleaners inherit from `Cleaner`, which loads the raw CSV (`data_folder`) into `origin_df` on construction and exposes a `processed_df` for results. Concrete cleaners such as `StreamflowCleaner` and `WaterlevelCleaner` operate on `self.origin_df` and write their cleaned result to `self.processed_df`. + +```python +from hydrodatasource.cleaner.streamflow_cleaner import StreamflowCleaner + +cleaner = StreamflowCleaner(data_folder="path/to/raw_flow.csv") +cleaner.anomaly_process(["moving_average"]) # writes a smoothed column into processed_df +``` + ## Rainfall Cleaner **File:** `rainfall_cleaner.py` @@ -29,10 +40,10 @@ The `cleaner` module provides a suite of tools for cleaning and preprocessing ra `StreamflowCleaner` focuses on smoothing noisy streamflow data. This is often necessary to reduce measurement noise without distorting the underlying hydrological signal. -It offers several smoothing algorithms, including: +It offers several smoothing algorithms, selected by the strings passed to `anomaly_process`: - Simple Moving Average (`moving_average`) -- Kalman Filter (`kalman_filter`) -- Low-pass Butterworth Filter (`lowpass_filter`) +- Kalman Filter (`kalman`) +- Low-pass Butterworth Filter (`lowpass`) - Fast Fourier Transform (`FFT`) and Wavelet (`wavelet`) filtering All methods are designed to be volume-preserving, meaning the total volume of streamflow is not changed by the smoothing process. diff --git a/docs/configs.md b/docs/configs.md index ae4eaf9..8d2de93 100644 --- a/docs/configs.md +++ b/docs/configs.md @@ -1,32 +1,60 @@ # Configuration -`hydrodatasource` uses a central YAML configuration file to manage data paths and connections to remote services. This makes it easy to adapt the library to your local environment. +`hydrodatasource` uses a central YAML configuration file to manage data paths and connections to remote services. It shares the same file and `storage.*` format as [hydrodataset](https://github.com/OuyangWenyu/hydrodataset) and `hydromodel`, so all three repositories resolve against one deterministic contract. ## The `hydro_setting.yml` File The configuration is managed through a file named `hydro_setting.yml` located in your user home directory (e.g., `C:\Users\YourUser` on Windows or `/home/YourUser` on Linux). -If this file does not exist, `hydrodatasource` will automatically create and use a default directory named `hydrodatasource_data` in your home directory. +If this file does not exist, `hydrodatasource` will use a default directory named `hydrodatasource_data` in your home directory (with a warning). -### Local Data Path Configuration - -For most use cases, you only need to configure the `local_data_path` section. This tells the library where to find and store your data. - -Here is an example of a minimal `hydro_setting.yml`: +### Unified `storage.*` Format ```yaml -local_data_path: - root: 'D:\data\hydro_data' # The main directory for all your data - datasets-origin: 'D:\data\hydro_data\origin' # For raw, unprocessed data - datasets-interim: 'D:\data\hydro_data\interim' # For intermediate, processed data - cache: 'D:\data\hydro_data\.cache' # For storing cached files like NetCDFs +storage: + default_source: local # 'local' or 'cloud' — the fallback when `source` is not given + local: + root: 'D:\data\hydrodatasource' # The main directory for all your data + cache: data\cache + s3: # Optional — cloud (MinIO/S3) access + endpoint_url: 'http://minio:9000' + key: 'access_key' + secret: 'secret_key' + bucket: hydro-data + prefix: hydromodel ``` -- **`root`**: The top-level directory for all project-related data. -- **`datasets-origin`**: This is where you should place your original, raw datasets. -- **`datasets-interim`**: This directory is used for storing data that has been processed or transformed in some way. -- **`cache`**: `hydrodatasource` uses this directory to store cached data, such as the NetCDF files generated by `SelfMadeHydroDataset`. This speeds up data loading on subsequent runs. +- **`storage.default_source`**: The storage backend used by `resolve_data_path()` / `open_dataset()` when the caller does not pass `source`. Either `local` or `cloud`. +- **`storage.local.root`**: The top-level directory for local data. Registered datasets resolve relative to this root. +- **`storage.cache`**: Where cached NetCDF files (generated by `SelfMadeHydroDataset`) are stored. This speeds up data loading on subsequent runs. +- **`storage.s3`**: Cloud (MinIO/S3) credentials and bucket. Used when `default_source: cloud` (or `source="cloud"`). + +## How Resolution Uses It + +`resolve_data_path(dataset_id)` / `open_dataset(dataset_id)` follow this contract, shared with [hydrodataset](https://github.com/OuyangWenyu/hydrodataset): + +1. Look up the dataset id in the registry (`HDS_DATASETS` + hydrodataset's datasets) to get the reader alias and relative path. +2. Resolve the storage root from `storage.local.root` (local) or `storage.s3` (cloud), per `source` / `default_source`. +3. Validate that the resulting absolute path exists. + +```python +from hydrodatasource.configs.data_resolver import ( + open_dataset, + resolve_data_path, + ResolverContext, +) + +# Uses ~/hydro_setting.yml storage.local.root (or storage.s3 when default_source: cloud) +uri = resolve_data_path("songliao_event") +ds = open_dataset("songliao_event") + +# Override the storage root without editing hydro_setting.yml +ctx = ResolverContext(storage={"local": {"root": "/your/data/root"}}) +ds = open_dataset("songliao_event", ctx=ctx) +``` -### Other Configurations +## Legacy Config -The `hydro_setting.yml` file can also be used to configure connections to a MinIO object storage server and a PostgreSQL database, but these are not required for basic local file-based operations. +The old `local_data_path.root` / `datasets-origin` / `datasets-interim` format, and the `minio.*` / +`postgres.*` blocks, are **no longer supported**. A non-empty `hydro_setting.yml` that is missing the +`storage` section is rejected with a warning and falls back to the default data root. diff --git a/docs/reader.md b/docs/reader.md index 2e296db..46839b4 100644 --- a/docs/reader.md +++ b/docs/reader.md @@ -1,17 +1,51 @@ # Reader -The `reader` module is the core component of `hydrodatasource` for accessing and reading various hydrological datasets. It provides a unified interface for handling different data sources, with a special focus on custom, user-prepared datasets. +The `reader` module is the core component of `hydrodatasource` for accessing and reading various hydrological datasets. It provides a **unified, URI-only** interface for handling different data sources, with a special focus on custom, user-prepared datasets. -## SelfMadeHydroDataset +## Two Ways to Open a Dataset -The `SelfMadeHydroDataset` class is the most important feature of the `reader` module. It allows you to read your own hydrological data as long as it follows a specific directory structure. This is designed for flexibility, enabling you to work with non-public or specially prepared datasets. +### 1. The `open_dataset()` factory (registered datasets) -### Directory Structure +If a dataset is registered in the resolution registry (either in `HDS_DATASETS` or one of hydrodataset's datasets), you can resolve and open it in one line: + +```python +from hydrodatasource.configs.data_resolver import open_dataset, resolve_data_path + +# Registered dataset — reads ~/hydro_setting.yml for the storage root +ds = open_dataset("songliao_event") + +# Or resolve just the path first +uri = resolve_data_path("songliao_event") +``` + +A custom `ResolverContext` can override the storage root and registry without touching `~/hydro_setting.yml`: + +```python +from hydrodatasource.configs.data_resolver import open_dataset, ResolverContext + +ctx = ResolverContext(storage={"local": {"root": "/your/data/root"}}) +ds = open_dataset("songliao_event", ctx=ctx) +``` + +### 2. Direct URI-only construction (custom datasets) + +For a dataset that is not registered, construct the reader class directly by passing the dataset directory as `uri`: + +```python +from hydrodatasource.reader.data_source import SelfMadeHydroDataset + +reader = SelfMadeHydroDataset(uri="/path/to/my_dataset", time_unit=["1D"]) +``` + +> **Note.** Legacy `data_path=` / `dataset_name=` constructor arguments were removed and now raise `ValueError`. +> Pass the absolute path (or `s3://` URI) of the dataset directory as `uri`. + +## Directory Structure To use `SelfMadeHydroDataset`, your data should be organized in the following structure: ``` -/path/to/your_dataset_name/ +/path/to/my_dataset/ ├── attributes/ │ ├── attributes.csv ├── shapes/ @@ -30,24 +64,23 @@ To use `SelfMadeHydroDataset`, your data should be organized in the following st - **`attributes/attributes.csv`**: A CSV file containing static attributes for each basin (e.g., area, slope, land cover). It must contain a `basin_id` column. - **`shapes/basins.shp`**: A shapefile containing the geographic boundaries of each basin. -- **`timeseries/`**: This directory holds the time series data, with subdirectories for each time resolution (e.g., `1D` for daily, `3h` for 3-hourly). +- **`timeseries/`**: Time series data, with subdirectories for each time resolution (`1h`, `3h`, `1D`, `8D`, `1M`). - Each subdirectory contains CSV files, one for each basin, named with the `basin_id`. - Each subdirectory also contains a `*_units_info.json` file that specifies the units for the variables in the CSV files. -### Example Usage +Extended readers may expect extra directories: -Here is how you can use `SelfMadeHydroDataset` to read your data: +- **`intermediate/`** — interval-basin data with topology (`TgHydroDatasource`). +- **`stations/`** — gauging-station data and adjacency matrices (`StationHydroDataset`). +- **`forecasts/`** — forecast time series (`SelfMadeForecastDataset`). + +## Example Usage ```python from hydrodatasource.reader.data_source import SelfMadeHydroDataset -# Path to the parent directory of your dataset -data_path = "/path/to/your_data/" -# The name of your dataset directory -dataset_name = "my_custom_dataset" - -# Initialize the reader -reader = SelfMadeHydroDataset(data_path=data_path, dataset_name=dataset_name, time_unit=["1D"]) +# Path to your dataset directory +reader = SelfMadeHydroDataset(uri="/path/to/my_dataset", time_unit=["1D"]) # Get a list of all basin IDs basin_ids = reader.read_object_ids() @@ -61,7 +94,7 @@ timeseries_data = reader.read_ts_xrdataset( gage_id_lst=basin_ids, t_range=t_range, var_lst=variables, - time_units=["1D"] + time_units=["1D"], ) # The result is a dictionary with time units as keys and xarray.Dataset as values @@ -69,7 +102,29 @@ daily_data = timeseries_data["1D"] print(daily_data) ``` +## Reader Aliases + +All `hydrodatasource` readers are registered in `READER_ALIASES`: + +| Alias | Class | Directory convention | +|-------|-------|---------------------| +| `selfmade` | `SelfMadeHydroDataset` | standard dataset | +| `longterm` | `LongTermDataset` | self-made dataset with long-term support | +| `forecast` | `SelfMadeForecastDataset` | standard + `forecasts/` | +| `station` | `StationHydroDataset` | standard + `stations/` | +| `tghydro` | `TgHydroDatasource` | standard + `intermediate/` + LSTM predictions | +| `floodevent` | `FloodEventDatasource` | flood-event data with per-basin event markers | +| `gages` | `Gages` | GAGES-II public dataset | +| `grdc` | `Grdc` | GRDC public dataset | +| `rainfall` | `RainfallReader` | cleaned station rainfall | +| `crd` | `Crd` | China reservoir database | +| `rsvrinflow` | `RsvrInflowReader` | reservoir inflow data | + +hydrodataset's public datasets (e.g. `camels_us`) are also resolvable through the same +`open_dataset()` / `resolve_data_path()` interface. + ## Other Readers -- **`SelfMadeForecastDataset`**: Extends `SelfMadeHydroDataset` to support forecast data, which is expected to be in a `forecasts` directory. -- **`StationHydroDataset`**: Extends `SelfMadeHydroDataset` to include data from gauging stations, which is expected to be in a `stations` directory. +- **`SelfMadeForecastDataset`**: Extends `SelfMadeHydroDataset` to support forecast data, expected in a `forecasts` directory. +- **`StationHydroDataset`**: Extends `SelfMadeHydroDataset` to include data from gauging stations, expected in a `stations` directory. +- **`TgHydroDatasource`**: Extends `SelfMadeHydroDataset` with LSTM prediction and graph-network structure support, using an `intermediate/` directory for interval-basin topology. diff --git a/docs/spec-unified-data-interface.md b/docs/spec-unified-data-interface.md deleted file mode 100644 index 78f0822..0000000 --- a/docs/spec-unified-data-interface.md +++ /dev/null @@ -1,188 +0,0 @@ -# Spec: Unified Data Interface Alignment (hydrodatasource) - -## Status - -Approved — ready for implementation. - -## Objective - -Align `hydrodatasource` with `hydromodel`'s [ADR 0001: Unified Data Path Resolution](../hydromodel/docs/adr/0001-unified-data-path-resolution.md) -and `hydrodataset`'s configuration system, so all three repositories share one deterministic -contract for resolving dataset identity → storage location → runtime URI. - -**Key principles:** -- hydrodatasource should reuse hydrodataset's config parsing code — NOT reinvent wheels -- hydrodatasource contributes its own **reader aliases** and **dataset registry**, which hydromodel consumes alongside hydrodataset's -- Database and legacy MinIO configuration must be **removed** — storage is unified under `storage.local` / `storage.s3` (cloud) -- All hydrodatasource reader classes must be registered as reader aliases - -## Resolved Questions - -| Question | Answer | -|----------|--------| -| Explicit hydrodataset dependency? | **Yes** — add to pyproject.toml since we import its settings.py and data_resolver.py directly | -| Old minio config? | **Remove** — superseded by `storage.s3` (cloud) | -| Old postgres config? | **Remove** — database access belongs in a real-time service repo, not coupled here | -| hydromodel dataset_dict.py? | **Deprecate/remove** — superseded by merged READER_ALIASES from both packages | -| Which readers to register? | **All of them** — every HydroData subclass gets a reader alias | - -## Current State - -### What exists today - -| Repo | data_resolver | datasets.yml | READER_ALIASES | settings.py | -|------|--------------|-------------|----------------|-------------| -| hydromodel | `hydromodel/configs/data_resolver.py` | `configs/datasets.yml` (2 datasets) | 4 entries (hardcoded) | built into data_resolver | -| hydrodataset | `hydrodataset/configs/data_resolver.py` | `configs/datasets.yml` (37 datasets) | 37 entries | `hydrodataset/configs/settings.py` | -| hydrodatasource | **NONE** | **NONE** | **NONE** | Old-style `configs/config.py` with import-time side effects | - -### The gap - -``` -OLD (current hydrodatasource): - ~/hydro_setting.yml (local_data_path.root, minio.*, postgres.*) - → configs/config.py (import-time side effects: SETTING, CACHE_DIR, FS, PS) - → reader classes (data_path + dataset_name) - -NEW (target): - ~/hydro_setting.yml (storage.local.root, storage.s3.* — NO minio, NO postgres) - → hydrodataset.configs.settings (pure functions) - + hydrodatasource.configs.data_resolver (READER_ALIASES + resolve_data_path) - + configs/datasets.yml (dataset id → reader + relative path) - → reader classes (accept resolved URI) -``` - -### Problems to fix - -1. Import-time side effects in `configs/config.py` -2. Old config format (`local_data_path.root` vs new `storage.local.root`) -3. No dataset registry (no `datasets.yml`) -4. No reader aliases for hydrodatasource's readers -5. Path-based constructors instead of URI-accepting ones -6. **minio config block — REMOVE** (cloud access via `storage.s3` only) -7. **postgres config block — REMOVE** (database decoupled from this repo) - -## Tech Stack - -- Python 3.10+ -- PyYAML for config parsing -- pathlib for path operations -- **hydrodataset** as explicit dependency in pyproject.toml - -## Commands - -```bash -uv sync --dev -uv run pytest tests/ -v -uv run black hydrodatasource/ -uv run flake8 hydrodatasource/ -uv build -``` - -## Design Decisions - -### Decision 1: Reuse hydrodataset's settings.py directly - -hydrodatasource will import `get_local_root`, `get_cache_dir`, `load_settings` from -`hydrodataset.configs.settings`. - -### Decision 2: hydrodatasource gets its own data_resolver.py - -`hydrodatasource/configs/data_resolver.py` with: -- `READER_ALIASES` for ALL hydrodatasource reader classes -- `resolve_data_path()` delegating to hydrodataset's logic -- `DatasetResolutionError` re-exported from hydrodataset - -### Decision 3: Two-stage migration for config.py - -**Stage A (this spec):** Add new functions, keep `CACHE_DIR`/`SETTING` globals via lazy compat layer. Remove minio/postgres globals. -**Stage B (future):** Deprecate and remove all import-time side effects. - -### Decision 4: Remove minio and postgres config blocks - -The `~/hydro_setting.yml` for hydrodatasource must NOT contain `minio.*` or `postgres.*`. -Cloud storage uses `storage.s3.*`. Database access is decoupled to a separate real-time service. - -### Decision 5: Register ALL reader classes - -All 11 HydroData subclasses get a reader alias. - -### Decision 6: READER_ALIASES format follows hydrodataset - -```python -READER_ALIASES: Dict[str, Dict[str, str]] = { - "alias_name": { - "module": "hydrodatasource.reader.floodevent", - "class": "FloodEventDatasource", - "category": "hydrodatasource", - }, -} -``` - -## Implementation Phases - -### Phase 1: Create data_resolver.py for hydrodatasource - -**File:** `hydrodatasource/configs/data_resolver.py` - -Register ALL reader classes: - -| Alias | Module | Class | -|-------|--------|-------| -| `floodevent` | hydrodatasource.reader.floodevent | FloodEventDatasource | -| `selfmade` | hydrodatasource.reader.data_source | SelfMadeHydroDataset | -| `longterm` | hydrodatasource.reader.data_source | LongTermDataset | -| `forecast` | hydrodatasource.reader.data_source | SelfMadeForecastDataset | -| `station` | hydrodatasource.reader.data_source | StationHydroDataset | -| `tghydro` | hydrodatasource.reader.data_source | TgHydroDatasource | -| `gages` | hydrodatasource.reader.gages | Gages | -| `grdc` | hydrodatasource.reader.grdc | Grdc | -| `rainfall` | hydrodatasource.reader.rainfall_reader | RainfallReader | -| `crd` | hydrodatasource.reader.reservoir_datasets | Crd | -| `rsvrinflow` | hydrodatasource.reader.rsvr_inflow_reader | RsvrInflowReader | - -Also implements `resolve_data_path(dataset_id, *, source, project_root)` and re-exports `DatasetResolutionError`. - -### Phase 2: Create top-level configs/datasets.yml - -**File:** `configs/datasets.yml` (repo root) - -### Phase 3: Refactor configs/config.py — additive + remove minio/postgres - -- Add lazy loading via `__getattr__` at module level -- Add `get_local_root()` → delegates to `hydrodataset.configs.settings` -- Add `get_cache_dir()` → same -- Keep `CACHE_DIR`, `SETTING`, `LOCAL_DATA_PATH` as deprecated compat -- **Remove:** `MINIO_PARAM`, `RO`, `S3`, `MC`, `FS` globals -- **Remove:** `POSTGRES_PARAM`, `PS`, `get_postgres_connection()` -- **Remove:** `read_setting()` validation of `minio` and `postgres` sections - -### Phase 4: Add URI-accepting constructors to reader classes - -Update `HydroData.__init__` and subclasses to accept `uri` parameter. - -### Phase 5: Update hydromodel data_resolver.py - -Import and merge READER_ALIASES from both hydrodataset and hydrodatasource. -Deprecate `dataset_dict.py`. - -### Phase 6: Add hydrodataset dependency to pyproject.toml - -## Success Criteria - -- [ ] `hydrodatasource/configs/data_resolver.py` with READER_ALIASES for ALL 11 reader classes -- [ ] `configs/datasets.yml` at repo root -- [ ] `resolve_data_path("songliao_event")` resolves correctly -- [ ] `from hydrodatasource.configs.config import CACHE_DIR` still works -- [ ] No `minio` or `postgres` config validation in config.py -- [ ] `SelfMadeHydroDataset(uri="/data/...")` works -- [ ] hydromodel can import and merge hydrodatasource's READER_ALIASES -- [ ] No import-time side effects in new code paths -- [ ] Same `~/hydro_setting.yml` (`storage.local.root`) works for all three repos -- [ ] hydrodataset is an explicit dependency in pyproject.toml - -## Boundaries - -- **Always do:** Use hydrodataset's settings.py; define reader aliases in Python code; validate paths per ADR 0001; use `storage.local.root` format -- **Ask first:** Removing old `config.py` globals entirely; changing reader class constructor signatures incompatibly -- **Never do:** Define Python module import paths in YAML; create datasets with absolute paths; allow `..` in dataset paths; commit secrets; add `minio.*` or `postgres.*` config validation back diff --git a/pyproject.toml b/pyproject.toml index 92a05ba..43602f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ dependencies = [ "pykalman", "xarray", "hydroutils>=0.2.0", - "hydrodataset>=0.2.0", + "hydrodataset>=0.3.0", "hydrotopo", "aqua-fetch[all]", ] From 02991f869d95f9fcfb1a1e1ded0d8d5c1829912b Mon Sep 17 00:00:00 2001 From: owen Date: Sat, 15 Aug 2026 19:42:46 +0800 Subject: [PATCH 30/34] ci: ignore local hydrodataset source in uv sync (--no-sources) The [tool.uv.sources] table points hydrodataset at a local editable path (../hydrodataset), which does not exist in CI runners. Add --no-sources so CI resolves hydrodataset>=0.3.0 from PyPI; local dev keeps the editable path. --- .github/workflows/build.yml | 2 +- .github/workflows/docs.yml | 2 +- .github/workflows/pypi.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 91cfc70..48dc314 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,7 +35,7 @@ jobs: - name: Install dependencies run: | - uv sync --dev + uv sync --no-sources --dev - name: Run tests run: | diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index f741297..5e575a1 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -15,6 +15,6 @@ jobs: - name: Setup Python run: uv python install 3.11 - name: Install dependencies - run: uv sync --all-extras + run: uv sync --no-sources --all-extras - name: Deploy documentation run: uv run mkdocs gh-deploy --force \ No newline at end of file diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index b84d8c3..3535f7b 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -24,7 +24,7 @@ jobs: run: uv python install 3.11 - name: Install dependencies - run: uv sync --all-extras # Install all dependencies including dev + run: uv sync --no-sources --all-extras # Install all dependencies including dev - name: Build package run: uv build From 4cd80833b6ab92b5703caf1acd9dfb4751cdc82c Mon Sep 17 00:00:00 2001 From: owen Date: Sat, 15 Aug 2026 19:44:26 +0800 Subject: [PATCH 31/34] ci: add --no-sources to uv run steps in build/docs workflows uv run also reads [tool.uv.sources]; add --no-sources so it resolves the project from the already-synced PyPI environment instead of the local hydrodataset path that does not exist on CI runners. --- .github/workflows/build.yml | 2 +- .github/workflows/docs.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 48dc314..01594b7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -39,4 +39,4 @@ jobs: - name: Run tests run: | - uv run pytest -m "not internal_data" + uv run --no-sources pytest -m "not internal_data" diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 5e575a1..7589c84 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -17,4 +17,4 @@ jobs: - name: Install dependencies run: uv sync --no-sources --all-extras - name: Deploy documentation - run: uv run mkdocs gh-deploy --force \ No newline at end of file + run: uv run --no-sources mkdocs gh-deploy --force \ No newline at end of file From cb6c990a6942c62700f941cddd7e69193ee06400 Mon Sep 17 00:00:00 2001 From: owen Date: Sat, 15 Aug 2026 19:48:25 +0800 Subject: [PATCH 32/34] fix: pin h5py<3.15 to avoid HDF5 wheel mismatch with netCDF4 1.7.3 netCDF4 1.7.2+ wheels bundle HDF5 1.14.6 which is incompatible with the h5py 3.15.x wheels, causing 'NetCDF: HDF error' when writing with to_netcdf on Linux CI. Per Unidata/netcdf4-python#1438, h5py 3.14.x is compatible with netCDF4 1.7.3. --- pyproject.toml | 5 +++- uv.lock | 67 ++++++++++++++++++-------------------------------- 2 files changed, 28 insertions(+), 44 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 43602f0..b38104a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,10 @@ dependencies = [ "kerchunk", "cfgrib", "eccodes", - "h5py", + # netCDF4 1.7.2+ wheels bundle an HDF5 (1.14.6) incompatible with h5py 3.15.x, + # causing "NetCDF: HDF error" on write on Linux. Pin h5py<3.15 to a compatible build. + # See https://github.com/Unidata/netcdf4-python/issues/1438 + "h5py<3.15", "chardet", "intake", "polars", diff --git a/uv.lock b/uv.lock index cc50955..83ec236 100644 --- a/uv.lock +++ b/uv.lock @@ -2262,53 +2262,34 @@ wheels = [ [[package]] name = "h5py" -version = "3.15.1" +version = "3.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "numpy", version = "2.3.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4d/6a/0d79de0b025aa85dc8864de8e97659c94cf3d23148394a954dc5ca52f8c8/h5py-3.15.1.tar.gz", hash = "sha256:c86e3ed45c4473564de55aa83b6fc9e5ead86578773dfbd93047380042e26b69", size = 426236, upload-time = "2025-10-16T10:35:27.404Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/86/30/8fa61698b438dd751fa46a359792e801191dadab560d0a5f1c709443ef8e/h5py-3.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:67e59f6c2f19a32973a40f43d9a088ae324fe228c8366e25ebc57ceebf093a6b", size = 3414477, upload-time = "2025-10-16T10:33:24.201Z" }, - { url = "https://files.pythonhosted.org/packages/16/16/db2f63302937337c4e9e51d97a5984b769bdb7488e3d37632a6ac297f8ef/h5py-3.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0e2f471688402c3404fa4e13466e373e622fd4b74b47b56cfdff7cc688209422", size = 2850298, upload-time = "2025-10-16T10:33:27.747Z" }, - { url = "https://files.pythonhosted.org/packages/fc/2e/f1bb7de9b05112bfd14d5206090f0f92f1e75bbb412fbec5d4653c3d44dd/h5py-3.15.1-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4c45802bcb711e128a6839cb6c01e9ac648dc55df045c9542a675c771f15c8d5", size = 4523605, upload-time = "2025-10-16T10:33:31.168Z" }, - { url = "https://files.pythonhosted.org/packages/05/8a/63f4b08f3628171ce8da1a04681a65ee7ac338fde3cb3e9e3c9f7818e4da/h5py-3.15.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64ce3f6470adb87c06e3a8dd1b90e973699f1759ad79bfa70c230939bff356c9", size = 4735346, upload-time = "2025-10-16T10:33:34.759Z" }, - { url = "https://files.pythonhosted.org/packages/74/48/f16d12d9de22277605bcc11c0dcab5e35f06a54be4798faa2636b5d44b3c/h5py-3.15.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4411c1867b9899a25e983fff56d820a66f52ac326bbe10c7cdf7d832c9dcd883", size = 4175305, upload-time = "2025-10-16T10:33:38.83Z" }, - { url = "https://files.pythonhosted.org/packages/d6/2f/47cdbff65b2ce53c27458c6df63a232d7bb1644b97df37b2342442342c84/h5py-3.15.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2cbc4104d3d4aca9d6db8c0c694555e255805bfeacf9eb1349bda871e26cacbe", size = 4653602, upload-time = "2025-10-16T10:33:42.188Z" }, - { url = "https://files.pythonhosted.org/packages/c3/28/dc08de359c2f43a67baa529cb70d7f9599848750031975eed92d6ae78e1d/h5py-3.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:01f55111ca516f5568ae7a7fc8247dfce607de331b4467ee8a9a6ed14e5422c7", size = 2873601, upload-time = "2025-10-16T10:33:45.323Z" }, - { url = "https://files.pythonhosted.org/packages/41/fd/8349b48b15b47768042cff06ad6e1c229f0a4bd89225bf6b6894fea27e6d/h5py-3.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5aaa330bcbf2830150c50897ea5dcbed30b5b6d56897289846ac5b9e529ec243", size = 3434135, upload-time = "2025-10-16T10:33:47.954Z" }, - { url = "https://files.pythonhosted.org/packages/c1/b0/1c628e26a0b95858f54aba17e1599e7f6cd241727596cc2580b72cb0a9bf/h5py-3.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c970fb80001fffabb0109eaf95116c8e7c0d3ca2de854e0901e8a04c1f098509", size = 2870958, upload-time = "2025-10-16T10:33:50.907Z" }, - { url = "https://files.pythonhosted.org/packages/f9/e3/c255cafc9b85e6ea04e2ad1bba1416baa1d7f57fc98a214be1144087690c/h5py-3.15.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:80e5bb5b9508d5d9da09f81fd00abbb3f85da8143e56b1585d59bc8ceb1dba8b", size = 4504770, upload-time = "2025-10-16T10:33:54.357Z" }, - { url = "https://files.pythonhosted.org/packages/8b/23/4ab1108e87851ccc69694b03b817d92e142966a6c4abd99e17db77f2c066/h5py-3.15.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5b849ba619a066196169763c33f9f0f02e381156d61c03e000bb0100f9950faf", size = 4700329, upload-time = "2025-10-16T10:33:57.616Z" }, - { url = "https://files.pythonhosted.org/packages/a4/e4/932a3a8516e4e475b90969bf250b1924dbe3612a02b897e426613aed68f4/h5py-3.15.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e7f6c841efd4e6e5b7e82222eaf90819927b6d256ab0f3aca29675601f654f3c", size = 4152456, upload-time = "2025-10-16T10:34:00.843Z" }, - { url = "https://files.pythonhosted.org/packages/2a/0a/f74d589883b13737021b2049ac796328f188dbb60c2ed35b101f5b95a3fc/h5py-3.15.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ca8a3a22458956ee7b40d8e39c9a9dc01f82933e4c030c964f8b875592f4d831", size = 4617295, upload-time = "2025-10-16T10:34:04.154Z" }, - { url = "https://files.pythonhosted.org/packages/23/95/499b4e56452ef8b6c95a271af0dde08dac4ddb70515a75f346d4f400579b/h5py-3.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:550e51131376889656feec4aff2170efc054a7fe79eb1da3bb92e1625d1ac878", size = 2882129, upload-time = "2025-10-16T10:34:06.886Z" }, - { url = "https://files.pythonhosted.org/packages/ce/bb/cfcc70b8a42222ba3ad4478bcef1791181ea908e2adbd7d53c66395edad5/h5py-3.15.1-cp311-cp311-win_arm64.whl", hash = "sha256:b39239947cb36a819147fc19e86b618dcb0953d1cd969f5ed71fc0de60392427", size = 2477121, upload-time = "2025-10-16T10:34:09.579Z" }, - { url = "https://files.pythonhosted.org/packages/62/b8/c0d9aa013ecfa8b7057946c080c0c07f6fa41e231d2e9bd306a2f8110bdc/h5py-3.15.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:316dd0f119734f324ca7ed10b5627a2de4ea42cc4dfbcedbee026aaa361c238c", size = 3399089, upload-time = "2025-10-16T10:34:12.135Z" }, - { url = "https://files.pythonhosted.org/packages/a4/5e/3c6f6e0430813c7aefe784d00c6711166f46225f5d229546eb53032c3707/h5py-3.15.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b51469890e58e85d5242e43aab29f5e9c7e526b951caab354f3ded4ac88e7b76", size = 2847803, upload-time = "2025-10-16T10:34:14.564Z" }, - { url = "https://files.pythonhosted.org/packages/00/69/ba36273b888a4a48d78f9268d2aee05787e4438557450a8442946ab8f3ec/h5py-3.15.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a33bfd5dfcea037196f7778534b1ff7e36a7f40a89e648c8f2967292eb6898e", size = 4914884, upload-time = "2025-10-16T10:34:18.452Z" }, - { url = "https://files.pythonhosted.org/packages/3a/30/d1c94066343a98bb2cea40120873193a4fed68c4ad7f8935c11caf74c681/h5py-3.15.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:25c8843fec43b2cc368aa15afa1cdf83fc5e17b1c4e10cd3771ef6c39b72e5ce", size = 5109965, upload-time = "2025-10-16T10:34:21.853Z" }, - { url = "https://files.pythonhosted.org/packages/81/3d/d28172116eafc3bc9f5991b3cb3fd2c8a95f5984f50880adfdf991de9087/h5py-3.15.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a308fd8681a864c04423c0324527237a0484e2611e3441f8089fd00ed56a8171", size = 4561870, upload-time = "2025-10-16T10:34:26.69Z" }, - { url = "https://files.pythonhosted.org/packages/a5/83/393a7226024238b0f51965a7156004eaae1fcf84aa4bfecf7e582676271b/h5py-3.15.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f4a016df3f4a8a14d573b496e4d1964deb380e26031fc85fb40e417e9131888a", size = 5037161, upload-time = "2025-10-16T10:34:30.383Z" }, - { url = "https://files.pythonhosted.org/packages/cf/51/329e7436bf87ca6b0fe06dd0a3795c34bebe4ed8d6c44450a20565d57832/h5py-3.15.1-cp312-cp312-win_amd64.whl", hash = "sha256:59b25cf02411bf12e14f803fef0b80886444c7fe21a5ad17c6a28d3f08098a1e", size = 2874165, upload-time = "2025-10-16T10:34:33.461Z" }, - { url = "https://files.pythonhosted.org/packages/09/a8/2d02b10a66747c54446e932171dd89b8b4126c0111b440e6bc05a7c852ec/h5py-3.15.1-cp312-cp312-win_arm64.whl", hash = "sha256:61d5a58a9851e01ee61c932bbbb1c98fe20aba0a5674776600fb9a361c0aa652", size = 2458214, upload-time = "2025-10-16T10:34:35.733Z" }, - { url = "https://files.pythonhosted.org/packages/88/b3/40207e0192415cbff7ea1d37b9f24b33f6d38a5a2f5d18a678de78f967ae/h5py-3.15.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c8440fd8bee9500c235ecb7aa1917a0389a2adb80c209fa1cc485bd70e0d94a5", size = 3376511, upload-time = "2025-10-16T10:34:38.596Z" }, - { url = "https://files.pythonhosted.org/packages/31/96/ba99a003c763998035b0de4c299598125df5fc6c9ccf834f152ddd60e0fb/h5py-3.15.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ab2219dbc6fcdb6932f76b548e2b16f34a1f52b7666e998157a4dfc02e2c4123", size = 2826143, upload-time = "2025-10-16T10:34:41.342Z" }, - { url = "https://files.pythonhosted.org/packages/6a/c2/fc6375d07ea3962df7afad7d863fe4bde18bb88530678c20d4c90c18de1d/h5py-3.15.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8cb02c3a96255149ed3ac811eeea25b655d959c6dd5ce702c9a95ff11859eb5", size = 4908316, upload-time = "2025-10-16T10:34:44.619Z" }, - { url = "https://files.pythonhosted.org/packages/d9/69/4402ea66272dacc10b298cca18ed73e1c0791ff2ae9ed218d3859f9698ac/h5py-3.15.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:121b2b7a4c1915d63737483b7bff14ef253020f617c2fb2811f67a4bed9ac5e8", size = 5103710, upload-time = "2025-10-16T10:34:48.639Z" }, - { url = "https://files.pythonhosted.org/packages/e0/f6/11f1e2432d57d71322c02a97a5567829a75f223a8c821764a0e71a65cde8/h5py-3.15.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:59b0d63b318bf3cc06687def2b45afd75926bbc006f7b8cd2b1a231299fc8599", size = 4556042, upload-time = "2025-10-16T10:34:51.841Z" }, - { url = "https://files.pythonhosted.org/packages/18/88/3eda3ef16bfe7a7dbc3d8d6836bbaa7986feb5ff091395e140dc13927bcc/h5py-3.15.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e02fe77a03f652500d8bff288cbf3675f742fc0411f5a628fa37116507dc7cc0", size = 5030639, upload-time = "2025-10-16T10:34:55.257Z" }, - { url = "https://files.pythonhosted.org/packages/e5/ea/fbb258a98863f99befb10ed727152b4ae659f322e1d9c0576f8a62754e81/h5py-3.15.1-cp313-cp313-win_amd64.whl", hash = "sha256:dea78b092fd80a083563ed79a3171258d4a4d307492e7cf8b2313d464c82ba52", size = 2864363, upload-time = "2025-10-16T10:34:58.099Z" }, - { url = "https://files.pythonhosted.org/packages/5d/c9/35021cc9cd2b2915a7da3026e3d77a05bed1144a414ff840953b33937fb9/h5py-3.15.1-cp313-cp313-win_arm64.whl", hash = "sha256:c256254a8a81e2bddc0d376e23e2a6d2dc8a1e8a2261835ed8c1281a0744cd97", size = 2449570, upload-time = "2025-10-16T10:35:00.473Z" }, - { url = "https://files.pythonhosted.org/packages/a0/2c/926eba1514e4d2e47d0e9eb16c784e717d8b066398ccfca9b283917b1bfb/h5py-3.15.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:5f4fb0567eb8517c3ecd6b3c02c4f4e9da220c8932604960fd04e24ee1254763", size = 3380368, upload-time = "2025-10-16T10:35:03.117Z" }, - { url = "https://files.pythonhosted.org/packages/65/4b/d715ed454d3baa5f6ae1d30b7eca4c7a1c1084f6a2edead9e801a1541d62/h5py-3.15.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:954e480433e82d3872503104f9b285d369048c3a788b2b1a00e53d1c47c98dd2", size = 2833793, upload-time = "2025-10-16T10:35:05.623Z" }, - { url = "https://files.pythonhosted.org/packages/ef/d4/ef386c28e4579314610a8bffebbee3b69295b0237bc967340b7c653c6c10/h5py-3.15.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fd125c131889ebbef0849f4a0e29cf363b48aba42f228d08b4079913b576bb3a", size = 4903199, upload-time = "2025-10-16T10:35:08.972Z" }, - { url = "https://files.pythonhosted.org/packages/33/5d/65c619e195e0b5e54ea5a95c1bb600c8ff8715e0d09676e4cce56d89f492/h5py-3.15.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:28a20e1a4082a479b3d7db2169f3a5034af010b90842e75ebbf2e9e49eb4183e", size = 5097224, upload-time = "2025-10-16T10:35:12.808Z" }, - { url = "https://files.pythonhosted.org/packages/30/30/5273218400bf2da01609e1292f562c94b461fcb73c7a9e27fdadd43abc0a/h5py-3.15.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa8df5267f545b4946df8ca0d93d23382191018e4cda2deda4c2cedf9a010e13", size = 4551207, upload-time = "2025-10-16T10:35:16.24Z" }, - { url = "https://files.pythonhosted.org/packages/d3/39/a7ef948ddf4d1c556b0b2b9559534777bccc318543b3f5a1efdf6b556c9c/h5py-3.15.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:99d374a21f7321a4c6ab327c4ab23bd925ad69821aeb53a1e75dd809d19f67fa", size = 5025426, upload-time = "2025-10-16T10:35:19.831Z" }, - { url = "https://files.pythonhosted.org/packages/b6/d8/7368679b8df6925b8415f9dcc9ab1dab01ddc384d2b2c24aac9191bd9ceb/h5py-3.15.1-cp314-cp314-win_amd64.whl", hash = "sha256:9c73d1d7cdb97d5b17ae385153472ce118bed607e43be11e9a9deefaa54e0734", size = 2865704, upload-time = "2025-10-16T10:35:22.658Z" }, - { url = "https://files.pythonhosted.org/packages/d3/b7/4a806f85d62c20157e62e58e03b27513dc9c55499768530acc4f4c5ce4be/h5py-3.15.1-cp314-cp314-win_arm64.whl", hash = "sha256:a6d8c5a05a76aca9a494b4c53ce8a9c29023b7f64f625c6ce1841e92a362ccdf", size = 2465544, upload-time = "2025-10-16T10:35:25.695Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/5d/57/dfb3c5c3f1bf5f5ef2e59a22dec4ff1f3d7408b55bfcefcfb0ea69ef21c6/h5py-3.14.0.tar.gz", hash = "sha256:2372116b2e0d5d3e5e705b7f663f7c8d96fa79a4052d250484ef91d24d6a08f4", size = 424323, upload-time = "2025-06-06T14:06:15.01Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/89/06cbb421e01dea2e338b3154326523c05d9698f89a01f9d9b65e1ec3fb18/h5py-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:24df6b2622f426857bda88683b16630014588a0e4155cba44e872eb011c4eaed", size = 3332522, upload-time = "2025-06-06T14:04:13.775Z" }, + { url = "https://files.pythonhosted.org/packages/c3/e7/6c860b002329e408348735bfd0459e7b12f712c83d357abeef3ef404eaa9/h5py-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6ff2389961ee5872de697054dd5a033b04284afc3fb52dc51d94561ece2c10c6", size = 2831051, upload-time = "2025-06-06T14:04:18.206Z" }, + { url = "https://files.pythonhosted.org/packages/fa/cd/3dd38cdb7cc9266dc4d85f27f0261680cb62f553f1523167ad7454e32b11/h5py-3.14.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:016e89d3be4c44f8d5e115fab60548e518ecd9efe9fa5c5324505a90773e6f03", size = 4324677, upload-time = "2025-06-06T14:04:23.438Z" }, + { url = "https://files.pythonhosted.org/packages/b1/45/e1a754dc7cd465ba35e438e28557119221ac89b20aaebef48282654e3dc7/h5py-3.14.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1223b902ef0b5d90bcc8a4778218d6d6cd0f5561861611eda59fa6c52b922f4d", size = 4557272, upload-time = "2025-06-06T14:04:28.863Z" }, + { url = "https://files.pythonhosted.org/packages/5c/06/f9506c1531645829d302c420851b78bb717af808dde11212c113585fae42/h5py-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:852b81f71df4bb9e27d407b43071d1da330d6a7094a588efa50ef02553fa7ce4", size = 2866734, upload-time = "2025-06-06T14:04:33.5Z" }, + { url = "https://files.pythonhosted.org/packages/61/1b/ad24a8ce846cf0519695c10491e99969d9d203b9632c4fcd5004b1641c2e/h5py-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f30dbc58f2a0efeec6c8836c97f6c94afd769023f44e2bb0ed7b17a16ec46088", size = 3352382, upload-time = "2025-06-06T14:04:37.95Z" }, + { url = "https://files.pythonhosted.org/packages/36/5b/a066e459ca48b47cc73a5c668e9924d9619da9e3c500d9fb9c29c03858ec/h5py-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:543877d7f3d8f8a9828ed5df6a0b78ca3d8846244b9702e99ed0d53610b583a8", size = 2852492, upload-time = "2025-06-06T14:04:42.092Z" }, + { url = "https://files.pythonhosted.org/packages/08/0c/5e6aaf221557314bc15ba0e0da92e40b24af97ab162076c8ae009320a42b/h5py-3.14.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c497600c0496548810047257e36360ff551df8b59156d3a4181072eed47d8ad", size = 4298002, upload-time = "2025-06-06T14:04:47.106Z" }, + { url = "https://files.pythonhosted.org/packages/21/d4/d461649cafd5137088fb7f8e78fdc6621bb0c4ff2c090a389f68e8edc136/h5py-3.14.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:723a40ee6505bd354bfd26385f2dae7bbfa87655f4e61bab175a49d72ebfc06b", size = 4516618, upload-time = "2025-06-06T14:04:52.467Z" }, + { url = "https://files.pythonhosted.org/packages/db/0c/6c3f879a0f8e891625817637fad902da6e764e36919ed091dc77529004ac/h5py-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d2744b520440a996f2dae97f901caa8a953afc055db4673a993f2d87d7f38713", size = 2874888, upload-time = "2025-06-06T14:04:56.95Z" }, + { url = "https://files.pythonhosted.org/packages/3e/77/8f651053c1843391e38a189ccf50df7e261ef8cd8bfd8baba0cbe694f7c3/h5py-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e0045115d83272090b0717c555a31398c2c089b87d212ceba800d3dc5d952e23", size = 3312740, upload-time = "2025-06-06T14:05:01.193Z" }, + { url = "https://files.pythonhosted.org/packages/ff/10/20436a6cf419b31124e59fefc78d74cb061ccb22213226a583928a65d715/h5py-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6da62509b7e1d71a7d110478aa25d245dd32c8d9a1daee9d2a42dba8717b047a", size = 2829207, upload-time = "2025-06-06T14:05:05.061Z" }, + { url = "https://files.pythonhosted.org/packages/3f/19/c8bfe8543bfdd7ccfafd46d8cfd96fce53d6c33e9c7921f375530ee1d39a/h5py-3.14.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:554ef0ced3571366d4d383427c00c966c360e178b5fb5ee5bb31a435c424db0c", size = 4708455, upload-time = "2025-06-06T14:05:11.528Z" }, + { url = "https://files.pythonhosted.org/packages/86/f9/f00de11c82c88bfc1ef22633557bfba9e271e0cb3189ad704183fc4a2644/h5py-3.14.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cbd41f4e3761f150aa5b662df991868ca533872c95467216f2bec5fcad84882", size = 4929422, upload-time = "2025-06-06T14:05:18.399Z" }, + { url = "https://files.pythonhosted.org/packages/7a/6d/6426d5d456f593c94b96fa942a9b3988ce4d65ebaf57d7273e452a7222e8/h5py-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:bf4897d67e613ecf5bdfbdab39a1158a64df105827da70ea1d90243d796d367f", size = 2862845, upload-time = "2025-06-06T14:05:23.699Z" }, + { url = "https://files.pythonhosted.org/packages/6c/c2/7efe82d09ca10afd77cd7c286e42342d520c049a8c43650194928bcc635c/h5py-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:aa4b7bbce683379b7bf80aaba68e17e23396100336a8d500206520052be2f812", size = 3289245, upload-time = "2025-06-06T14:05:28.24Z" }, + { url = "https://files.pythonhosted.org/packages/4f/31/f570fab1239b0d9441024b92b6ad03bb414ffa69101a985e4c83d37608bd/h5py-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ef9603a501a04fcd0ba28dd8f0995303d26a77a980a1f9474b3417543d4c6174", size = 2807335, upload-time = "2025-06-06T14:05:31.997Z" }, + { url = "https://files.pythonhosted.org/packages/0d/ce/3a21d87896bc7e3e9255e0ad5583ae31ae9e6b4b00e0bcb2a67e2b6acdbc/h5py-3.14.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8cbaf6910fa3983c46172666b0b8da7b7bd90d764399ca983236f2400436eeb", size = 4700675, upload-time = "2025-06-06T14:05:37.38Z" }, + { url = "https://files.pythonhosted.org/packages/e7/ec/86f59025306dcc6deee5fda54d980d077075b8d9889aac80f158bd585f1b/h5py-3.14.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d90e6445ab7c146d7f7981b11895d70bc1dd91278a4f9f9028bc0c95e4a53f13", size = 4921632, upload-time = "2025-06-06T14:05:43.464Z" }, + { url = "https://files.pythonhosted.org/packages/3f/6d/0084ed0b78d4fd3e7530c32491f2884140d9b06365dac8a08de726421d4a/h5py-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:ae18e3de237a7a830adb76aaa68ad438d85fe6e19e0d99944a3ce46b772c69b3", size = 2852929, upload-time = "2025-06-06T14:05:47.659Z" }, ] [[package]] @@ -2478,7 +2459,7 @@ requires-dist = [ { name = "eccodes" }, { name = "flake8", marker = "extra == 'dev'" }, { name = "geopandas" }, - { name = "h5py" }, + { name = "h5py", specifier = "<3.15" }, { name = "hydrodataset", editable = "../hydrodataset" }, { name = "hydrotopo" }, { name = "hydroutils", specifier = ">=0.2.0" }, From dd2c30ff5ff76b737ac9640b7bce615d4a0b3eb4 Mon Sep 17 00:00:00 2001 From: owen Date: Sat, 15 Aug 2026 19:52:12 +0800 Subject: [PATCH 33/34] fix: make tomllib import Python 3.10-compatible in compat test tomllib is Python 3.11+; fall back to the tomli backport on 3.10 and add tomli as a conditional dev dependency so the pyproject assertion test collects on Python 3.10 (project supports 3.10+). --- pyproject.toml | 1 + tests/test_hydroutils_compat.py | 6 +++++- uv.lock | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b38104a..68fcc82 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,7 @@ dev = [ "pytest-asyncio", "pytest-mock", "pytest-dotenv", + "tomli; python_version < '3.11'", # tomllib backport for Python 3.10 # Packaging "wheel", diff --git a/tests/test_hydroutils_compat.py b/tests/test_hydroutils_compat.py index d647a8d..509b967 100644 --- a/tests/test_hydroutils_compat.py +++ b/tests/test_hydroutils_compat.py @@ -26,9 +26,13 @@ import importlib import logging -import tomllib from pathlib import Path +try: + import tomllib # Python 3.11+ +except ModuleNotFoundError: # Python 3.10 + import tomli as tomllib + from packaging.requirements import Requirement diff --git a/uv.lock b/uv.lock index 83ec236..5f94a3b 100644 --- a/uv.lock +++ b/uv.lock @@ -2416,6 +2416,7 @@ dev = [ { name = "pytest-dotenv" }, { name = "pytest-mock" }, { name = "pytest-runner" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, { name = "tox" }, { name = "twine" }, { name = "watchdog" }, @@ -2497,6 +2498,7 @@ requires-dist = [ { name = "scipy" }, { name = "sphinx", marker = "extra == 'docs'" }, { name = "sqlalchemy" }, + { name = "tomli", marker = "python_full_version < '3.11' and extra == 'dev'" }, { name = "tox", marker = "extra == 'dev'" }, { name = "twine", marker = "extra == 'dev'" }, { name = "watchdog", marker = "extra == 'dev'" }, From d87992adadb6b5bb182f4159435c5b1b44029cea Mon Sep 17 00:00:00 2001 From: owen Date: Sat, 15 Aug 2026 20:40:48 +0800 Subject: [PATCH 34/34] fix: address review findings on resolver robustness and cleanup config: inject a storage default when the storage section is missing (S4); correct the lazy-load docstring to note module-level binds still trigger init (I1) data_resolver: docstring examples use registered dataset ids (S1); document that resolution requires a storage root and raises DatasetResolutionError without config (S2) data_source: add NetCDF attributes fallback for S3 datasets, guarding the unconfigured-S3 case (S3); drop the dead download param from LongTermDataset (S6); remove stale data_path comment (N1) utils: guard is_minio_folder against an unconfigured S3 filesystem (S5) floodevent: fix stray spaces in the time-range f-string (N2) readers: remove dead pathlib imports left by the migration (I2) tests: decouple the package-export count floor from hydrodataset's registry (N4); add regression tests for review cleanup, suggestions, and S3 fallback --- hydrodatasource/configs/config.py | 4 + hydrodatasource/configs/data_resolver.py | 8 +- hydrodatasource/reader/data_source.py | 9 +- hydrodatasource/reader/floodevent.py | 2 +- hydrodatasource/reader/rainfall_reader.py | 1 - hydrodatasource/reader/reservoir_datasets.py | 1 - hydrodatasource/reader/rsvr_inflow_reader.py | 1 - hydrodatasource/utils/utils.py | 3 + tests/test_fix_review_cleanup.py | 140 +++++++++++++++++++ tests/test_fix_review_suggestions.py | 107 ++++++++++++++ tests/test_fix_s3_attr_fallback.py | 94 +++++++++++++ tests/test_package_exports.py | 4 +- 12 files changed, 364 insertions(+), 10 deletions(-) create mode 100644 tests/test_fix_review_cleanup.py create mode 100644 tests/test_fix_review_suggestions.py create mode 100644 tests/test_fix_s3_attr_fallback.py diff --git a/hydrodatasource/configs/config.py b/hydrodatasource/configs/config.py index 3bd6beb..50b4a3b 100644 --- a/hydrodatasource/configs/config.py +++ b/hydrodatasource/configs/config.py @@ -14,6 +14,9 @@ Critical globals (SETTING, CACHE_DIR, FS, LOCAL_ROOT, MINIO_PARAM) are lazy-loaded via module __getattr__ to avoid import-time side effects. +Note: a module-level ``from .config import CACHE_DIR`` still triggers this lazy +init at import time; prefer attribute access (e.g. ``conf.CACHE_DIR``) at call +sites for true deferral. FilePath: \\hydrodatasource\\hydrodatasource\\configs\\config.py Copyright (c) 2023-2026 Wenyu Ouyang. All rights reserved. @@ -87,6 +90,7 @@ def _init_settings() -> None: f"falling back to default data root: {default_root}", stacklevel=2, ) + setting["storage"] = {"local": {"root": default_root}} root = setting.get("storage", {}).get("local", {}).get("root", default_root) diff --git a/hydrodatasource/configs/data_resolver.py b/hydrodatasource/configs/data_resolver.py index 0d4ca2d..5a8e895 100644 --- a/hydrodatasource/configs/data_resolver.py +++ b/hydrodatasource/configs/data_resolver.py @@ -5,6 +5,12 @@ resolve_data_path() and open_dataset() following the same contract as hydromodel's ADR 0001. +Note: resolution requires a storage root. Provide a ~/hydro_setting.yml with +``storage.local.root`` (or ``storage.s3`` for cloud), or pass a +``ResolverContext(storage={...})``. With no config, resolution raises +``DatasetResolutionError`` (unlike ``configs.config`` which falls back to a +default home data root). + This module is a thin wrapper around hydrodataset.configs.data_resolver. It only uses the PUBLIC API of hydrodataset (resolve_data_path, open_dataset, READER_ALIASES, DatasetResolutionError). The hydrodatasource-specific parts are @@ -250,7 +256,7 @@ def open_dataset( -------- >>> ds = open_dataset("songliao_event") >>> ds = open_dataset("camels_us", source="cloud") - >>> ds = open_dataset("selfmade_basin", time_unit=["1D"]) + >>> ds = open_dataset("songliao_event", time_unit=["1D"]) """ return _hd_open_dataset( dataset_id, source=source, ctx=_with_hds_extras(ctx), **reader_kwargs diff --git a/hydrodatasource/reader/data_source.py b/hydrodatasource/reader/data_source.py index 3046efe..24b2636 100644 --- a/hydrodatasource/reader/data_source.py +++ b/hydrodatasource/reader/data_source.py @@ -103,7 +103,7 @@ def __init__( raise ValueError( "time_unit must be one of ['1h', '3h', '1D', '8D','1M']. We only support these time units now." ) - # Determine filesystem head from URI or data_path + # Determine filesystem head from URI effective_path = str(uri) if uri is not None else "" self.head = "minio" if "s3://" in effective_path else "local" super().__init__(uri=uri) @@ -171,7 +171,10 @@ def _where_attr_file(self, attr_dir): compatibility, otherwise fall back to a NetCDF file in the dir.""" attr_csv = os.path.join(attr_dir, "attributes.csv") if "s3://" in attr_dir: - return attr_csv + if conf.FS is None or conf.FS.exists(attr_csv): + return attr_csv + nc_files = sorted(conf.FS.glob(os.path.join(attr_dir, "*.nc"))) + return nc_files[0] if nc_files else attr_csv if os.path.exists(attr_csv): return attr_csv nc_files = sorted(glob.glob(os.path.join(attr_dir, "*.nc"))) @@ -896,7 +899,7 @@ def read_mean_prcp(self, gage_id_lst=None, unit="mm/d"): class LongTermDataset(SelfMadeHydroDataset): - def __init__(self, uri=None, download=False, time_unit=None, **kwargs): + def __init__(self, uri=None, time_unit=None, **kwargs): if time_unit is None: time_unit = ["1M"] super().__init__( diff --git a/hydrodatasource/reader/floodevent.py b/hydrodatasource/reader/floodevent.py index c010026..3c22e16 100644 --- a/hydrodatasource/reader/floodevent.py +++ b/hydrodatasource/reader/floodevent.py @@ -944,7 +944,7 @@ def read_ts_xrdataset_augmented( # 如果切片后的数据集为空，返回警告 if len(ds.time) == 0: print( - f"Warning: The specified time range {t_range [0]} to {t_range [1]} does not overlap with the dataset time range {ds_start_time} to {ds_end_time}" + f"Warning: The specified time range {t_range[0]} to {t_range[1]} does not overlap with the dataset time range {ds_start_time} to {ds_end_time}" ) # 应用变量过滤 diff --git a/hydrodatasource/reader/rainfall_reader.py b/hydrodatasource/reader/rainfall_reader.py index 5b4806a..4503122 100644 --- a/hydrodatasource/reader/rainfall_reader.py +++ b/hydrodatasource/reader/rainfall_reader.py @@ -9,7 +9,6 @@ import collections import os -from pathlib import Path import pandas as pd from tqdm import tqdm diff --git a/hydrodatasource/reader/reservoir_datasets.py b/hydrodatasource/reader/reservoir_datasets.py index b2043e0..b637a6d 100644 --- a/hydrodatasource/reader/reservoir_datasets.py +++ b/hydrodatasource/reader/reservoir_datasets.py @@ -10,7 +10,6 @@ import os import collections -from pathlib import Path import geopandas as gpd from hydrodatasource.reader.data_source import HydroData diff --git a/hydrodatasource/reader/rsvr_inflow_reader.py b/hydrodatasource/reader/rsvr_inflow_reader.py index 06c4a88..96c36a1 100644 --- a/hydrodatasource/reader/rsvr_inflow_reader.py +++ b/hydrodatasource/reader/rsvr_inflow_reader.py @@ -10,7 +10,6 @@ import collections import os -from pathlib import Path import pandas as pd from tqdm import tqdm diff --git a/hydrodatasource/utils/utils.py b/hydrodatasource/utils/utils.py index f24dea2..fbbb8af 100644 --- a/hydrodatasource/utils/utils.py +++ b/hydrodatasource/utils/utils.py @@ -134,6 +134,9 @@ def is_minio_folder(minio_url): True if the folder exists, False otherwise """ + if FS is None: + # S3 not configured in this environment; nothing can be a MinIO folder + return False if not FS.exists(minio_url): raise FileNotFoundError(f"No file or folder found in {minio_url}") if minio_url.endswith("/"): diff --git a/tests/test_fix_review_cleanup.py b/tests/test_fix_review_cleanup.py new file mode 100644 index 0000000..d02b842 --- /dev/null +++ b/tests/test_fix_review_cleanup.py @@ -0,0 +1,140 @@ +"""Guard tests: review findings from the unified-data-interface cleanup pass. + +Purpose +------- +A code review of the ``feat/unified-data-interface`` work surfaced three +hygiene issues. These tests guard the invariant that the issues stay fixed. +They are source-inspection tests (same style as ``test_fix_dead_code.py``): +they read the relevant ``.py`` files and assert on exact strings / the real +dataset registry. They deliberately avoid marking ``internal_data`` and never +import the package at module scope, so collection is side-effect free. + +Issue inventory +--------------- +A. Dead ``from pathlib import Path`` imports in three reader modules. The + symbol ``Path`` is never used in any of them (only the module docstring's + ``FilePath:`` header contains the substring "Path"). The import should be + deleted. Guard: the exact import statement must be absent. + +B. ``hydrodatasource/configs/data_resolver.py`` ``open_dataset`` docstring + example uses the unregistered dataset id ``selfmade_basin``. The only id + registered by hydrodatasource itself is ``songliao_event`` (``HDS_DATASETS``); + ``camels_us`` etc. come from hydrodataset's default registry. Calling + ``open_dataset("selfmade_basin", ...)`` would raise ``DatasetResolutionError``. + The example must use a registered id. Guard: every dataset id appearing in an + ``open_dataset("...")`` docstring example is a member of the real merged + registry, and ``selfmade_basin`` appears nowhere. + +C. (not guarded here) ``hydrodatasource/configs/config.py`` docstring wording + about lazy-loading "avoid import-time side effects". A source-inspection + assertion on docstring prose is too brittle, so it is intentionally skipped; + the fix is left to the implementer's judgment. +""" + +from pathlib import Path + +# Paths are resolved at import time from this file's location; this does not +# import the package, so it is collection-safe. +_REPO_ROOT = Path(__file__).resolve().parents[1] +_RAINFALL_READER_PATH = ( + _REPO_ROOT / "hydrodatasource" / "reader" / "rainfall_reader.py" +) +_RESERVOIR_DATASETS_PATH = ( + _REPO_ROOT / "hydrodatasource" / "reader" / "reservoir_datasets.py" +) +_RSVR_INFLOW_READER_PATH = ( + _REPO_ROOT / "hydrodatasource" / "reader" / "rsvr_inflow_reader.py" +) +_DATA_RESOLVER_PATH = ( + _REPO_ROOT / "hydrodatasource" / "configs" / "data_resolver.py" +) + + +def _source(path: Path) -> str: + """Return the current UTF-8 text of a source file.""" + return path.read_text(encoding="utf-8") + + +def _docstring_example_dataset_ids() -> list[str]: + """Extract dataset ids from ``open_dataset("...")`` docstring examples. + + Only the ``open_dataset`` call in the module docstring's Examples section is + inspected; the module-level ``open_dataset`` function is not invoked. + """ + import re + + source = _source(_DATA_RESOLVER_PATH) + return re.findall(r'open_dataset\("([^"]+)"', source) + + +# --------------------------------------------------------------------------- +# A. Dead `from pathlib import Path` imports in reader modules +# --------------------------------------------------------------------------- + + +def test_rainfall_reader_no_dead_pathlib_import(): + """``Path`` is unused in rainfall_reader.py, so its import must be gone.""" + assert "from pathlib import Path" not in _source(_RAINFALL_READER_PATH), ( + "hydrodatasource/reader/rainfall_reader.py still imports Path, but the " + "symbol is never used (the only 'Path' occurrence is the module " + "docstring's FilePath: header)." + ) + + +def test_reservoir_datasets_no_dead_pathlib_import(): + """``Path`` is unused in reservoir_datasets.py, so its import must be gone.""" + assert "from pathlib import Path" not in _source(_RESERVOIR_DATASETS_PATH), ( + "hydrodatasource/reader/reservoir_datasets.py still imports Path, but " + "the symbol is never used (the only 'Path' occurrence is the module " + "docstring's FilePath: header)." + ) + + +def test_rsvr_inflow_reader_no_dead_pathlib_import(): + """``Path`` is unused in rsvr_inflow_reader.py, so its import must be gone.""" + assert "from pathlib import Path" not in _source(_RSVR_INFLOW_READER_PATH), ( + "hydrodatasource/reader/rsvr_inflow_reader.py still imports Path, but " + "the symbol is never used (the only 'Path' occurrence is the module " + "docstring's FilePath: header)." + ) + + +# --------------------------------------------------------------------------- +# B. data_resolver.py docstring examples must use registered dataset ids +# --------------------------------------------------------------------------- + + +def test_data_resolver_docstring_examples_do_not_use_selfmade_basin(): + """The docstring must not reference the unregistered ``selfmade_basin`` id.""" + assert "selfmade_basin" not in _source(_DATA_RESOLVER_PATH), ( + "hydrodatasource/configs/data_resolver.py docstring example " + 'open_dataset("selfmade_basin", ...) uses an id that is registered ' + "neither in HDS_DATASETS nor in hydrodataset's default registry; " + "calling it would raise DatasetResolutionError. Use a registered id " + "such as 'songliao_event' or 'camels_us'." + ) + + +def test_data_resolver_docstring_examples_use_registered_dataset_ids(): + """Every ``open_dataset(\"...\")`` example id must exist in the real registry. + + The registered set is computed at runtime from hydrodataset's default + registry plus ``HDS_DATASETS`` (hydrodatasource's own entries), so the + check tracks registry changes instead of a hardcoded whitelist. + """ + from hydrodataset.configs.data_resolver import _DEFAULT_REGISTRY + from hydrodatasource.configs.data_resolver import HDS_DATASETS + + registered = set(_DEFAULT_REGISTRY) | set(HDS_DATASETS) + + example_ids = _docstring_example_dataset_ids() + assert example_ids, ( + "hydrodatasource/configs/data_resolver.py docstring must contain " + "open_dataset(\"\") examples to inspect." + ) + for dataset_id in example_ids: + assert dataset_id in registered, ( + f"data_resolver.py docstring example uses unregistered dataset id " + f"{dataset_id!r}; known registered ids include 'songliao_event' " + "and 'camels_us'." + ) diff --git a/tests/test_fix_review_suggestions.py b/tests/test_fix_review_suggestions.py new file mode 100644 index 0000000..aae3123 --- /dev/null +++ b/tests/test_fix_review_suggestions.py @@ -0,0 +1,107 @@ +"""Guard tests for three review Suggestion-level issues (RED on current code). + +These are reproduction tests for bugs surfaced in a code review pass. Each +test exercises the *unfixed* behaviour and must FAIL on the current source, +acting as the regression guard for the implementer's fix. + +S5 -- ``hydrodatasource/utils/utils.py::is_minio_folder`` missing ``FS is None`` + guard. Its sibling ``minio_file_list`` already tolerates an unconfigured S3 + backend (``if FS is None: return []``), but ``is_minio_folder`` calls + ``FS.exists(...)`` unconditionally, so with S3 unconfigured it raises + ``AttributeError: 'NoneType' object has no attribute 'exists'`` instead of + reporting the path is not a MinIO folder. Expected fix: return ``False`` + when ``FS`` is ``None``. + +S4 -- ``hydrodatasource/configs/config.py::_init_settings`` does not + normalise ``SETTING`` when the settings dict is non-empty but lacks a + ``storage`` key. The empty-settings branch injects + ``{"storage": {"local": {"root": ...}}}``, but the ``elif "storage" not in + setting`` branch only warns and leaves the dict without ``storage``, so + downstream ``SETTING["storage"]`` raises ``KeyError``. Expected fix: the + elif branch must inject the same ``storage.local.root`` fallback. + +S6 -- ``hydrodatasource/reader/data_source.py::LongTermDataset`` exposes a dead + ``download`` constructor parameter. ``__init__`` accepts ``download=False`` + but never reads it. Expected fix: remove the parameter (no in-repo caller + passes it). Test asserts the constructor signature no longer contains it. + +These tests deliberately avoid the ``internal_data`` marker and keep every +import inside the test function, so collection is side-effect free. +""" + +# ── S5: is_minio_folder must tolerate FS = None (S3 unconfigured) ─────────── + + +def test_is_minio_folder_returns_false_when_s3_not_configured(monkeypatch): + """is_minio_folder returns False (not AttributeError) when FS is None. + + The current code calls ``FS.exists(minio_url)`` unconditionally; with S3 + unconfigured ``FS`` is ``None`` and the call dies with AttributeError. The + expected behaviour, matching ``minio_file_list``'s ``if FS is None: return + []``, is that an unconfigured S3 backend means no path is a MinIO folder. + """ + import hydrodatasource.utils.utils as utils + + # Patch the module-level binding in utils' own namespace. utils.py does + # ``from ..configs.config import FS`` at import time, so patching + # config.FS would NOT reach the global ``FS`` that is_minio_folder reads + # (the module-binding trap) -- we must patch utils.FS directly. + monkeypatch.setattr(utils, "FS", None) + + assert utils.is_minio_folder("s3://bucket/some/dir") is False + + +# ── S4: _init_settings must normalise SETTING when 'storage' is missing ───── + + +def test_init_settings_injects_storage_defaults_when_storage_section_missing( + monkeypatch, +): + """A non-empty settings dict without a 'storage' key still yields SETTING['storage']. + + The empty-settings branch injects ``{"storage": {"local": {"root": ...}}}``, + but the elif branch (non-empty, no 'storage' key) only warns and leaves the + dict without 'storage' -- so ``SETTING["storage"]`` KeyErrors downstream. The + expected fix normalises the dict just like the empty branch does. + """ + from hydrodatasource.configs import config + + original_load = config._load_settings_from_file + + def mock_load(): + # Non-empty but missing the 'storage' section. + return {"project": "demo", "some": "legacy-key"} + + monkeypatch.setattr(config, "_load_settings_from_file", mock_load) + try: + config._init_settings() + + storage = config.SETTING["storage"] # KeyError on current code + assert storage["local"]["root"], ( + "storage.local.root must have a fallback value when the settings " + "file has no storage section" + ) + finally: + # Restore the real loader and re-initialize so other tests see a + # genuine config even when this test fails (RED). + monkeypatch.setattr(config, "_load_settings_from_file", original_load) + config._init_settings() + + +# ── S6: LongTermDataset must not expose the dead 'download' parameter ─────── + + +def test_longterm_dataset_init_has_no_download_parameter(): + """LongTermDataset.__init__ does not accept a dead 'download' argument. + + The constructor declares ``download=False`` but never reads it, so it is + pure dead weight on the unified URI-only constructor contract. No in-repo + caller passes ``download=`` (verified by searching the repository), so the + parameter can be dropped outright. + """ + import inspect + + from hydrodatasource.reader.data_source import LongTermDataset + + params = list(inspect.signature(LongTermDataset.__init__).parameters) + assert "download" not in params diff --git a/tests/test_fix_s3_attr_fallback.py b/tests/test_fix_s3_attr_fallback.py new file mode 100644 index 0000000..2fc527a --- /dev/null +++ b/tests/test_fix_s3_attr_fallback.py @@ -0,0 +1,94 @@ +"""Regression tests for the S3 branch of +``hydrodatasource.reader.data_source.SelfMadeHydroDataset._where_attr_file``. + +Defect (from review): the S3 branch returned ``attributes.csv`` unconditionally, +so an S3 dataset that stores its attributes as NetCDF could never be located. +The local branch already falls back to a ``*.nc`` file when ``attributes.csv`` +is absent; the S3 branch must mirror that fallback using ``conf.FS`` +(``exists`` / ``glob``) instead of ``os.path.exists`` / ``glob.glob``. + +Expected fixed behavior for ``"s3://" in attr_dir``:: + + if conf.FS.exists(attr_csv): + return attr_csv + nc_files = sorted(conf.FS.glob(os.path.join(attr_dir, "*.nc"))) + return nc_files[0] if nc_files else attr_csv + +RED on current code: ``test_s3_falls_back_to_first_nc_when_no_csv`` (case 2). +The other two cases are current-behavior guards that stay GREEN. + +The method uses no instance state, so each test calls it on an uninitialized +instance obtained via ``SelfMadeHydroDataset.__new__(...)``. ``conf.FS`` is +patched with a tiny inline fake exposing only ``exists`` / ``glob``. +""" + + +def _make_fake_fs(exists_result, glob_result): + """Return a minimal S3-filesystem stand-in for ``conf.FS``. + + ``exists`` answers a canned boolean; ``glob`` returns a canned list of + matching paths (as a real ``s3fs`` ``glob`` would). + """ + + class _FakeFS: + def exists(self, path): + return exists_result + + def glob(self, pattern): + return glob_result + + return _FakeFS() + + +def test_s3_returns_attributes_csv_when_it_exists(monkeypatch): + """S3 with an attributes.csv present -> attributes.csv is returned.""" + import os + + from hydrodatasource.reader.data_source import SelfMadeHydroDataset + + attr_dir = "s3://bucket/attrs" + attr_csv = os.path.join(attr_dir, "attributes.csv") + + monkeypatch.setattr( + "hydrodatasource.configs.config.FS", + _make_fake_fs(exists_result=True, glob_result=[]), + ) + + ds = SelfMadeHydroDataset.__new__(SelfMadeHydroDataset) + assert ds._where_attr_file(attr_dir) == attr_csv + + +def test_s3_falls_back_to_first_nc_when_no_csv(monkeypatch): + """S3 without attributes.csv but with a *.nc -> the .nc file is returned.""" + import os + + from hydrodatasource.reader.data_source import SelfMadeHydroDataset + + attr_dir = "s3://bucket/attrs" + nc_file = "s3://bucket/attrs/attrs.nc" + + monkeypatch.setattr( + "hydrodatasource.configs.config.FS", + _make_fake_fs(exists_result=False, glob_result=[nc_file]), + ) + + ds = SelfMadeHydroDataset.__new__(SelfMadeHydroDataset) + assert ds._where_attr_file(attr_dir) == nc_file + + +def test_s3_returns_attributes_csv_when_neither_csv_nor_nc_exists(monkeypatch): + """S3 with neither attributes.csv nor any *.nc -> attributes.csv is returned.""" + import os + + from hydrodatasource.reader.data_source import SelfMadeHydroDataset + + attr_dir = "s3://bucket/attrs" + attr_csv = os.path.join(attr_dir, "attributes.csv") + + monkeypatch.setattr( + "hydrodatasource.configs.config.FS", + _make_fake_fs(exists_result=False, glob_result=[]), + ) + + ds = SelfMadeHydroDataset.__new__(SelfMadeHydroDataset) + assert ds._where_attr_file(attr_dir) == attr_csv diff --git a/tests/test_package_exports.py b/tests/test_package_exports.py index e50806d..d0d3407 100644 --- a/tests/test_package_exports.py +++ b/tests/test_package_exports.py @@ -74,8 +74,8 @@ def test_merged_total_count(self): # 并集恒等式：每个别名恰好属于一个类别 assert len(READER_ALIASES) == len(hds) + len(hd) - # 防倒退下限：别名总数不得少于上一版本 - assert len(READER_ALIASES) >= 37 + # hydrodataset 的别名数量由 hydrodataset 自己维护；本仓库只保证自己的 11 个别名 + # （见 test_hydrodatasource_aliases_total_count）与并集恒等式成立。 def test_resolver_context_importable_from_package(self): """ResolverContext is importable from hydrodatasource."""