diff --git a/flopy4/cli.py b/flopy4/cli.py index 709dba53..069fef79 100644 --- a/flopy4/cli.py +++ b/flopy4/cli.py @@ -22,7 +22,7 @@ def _cmd_sync(args: argparse.Namespace) -> None: if result.installed: print(f"Installed {result.installed}") if result.removed: - print(f"Removed {len(result.removed)} orphaned module(s)") + print(f"Removed {len(result.removed)} stale module(s)") if result.version != "unknown": print(f"Synced flopy4.mf6 to MF6 version: {result.version}") diff --git a/flopy4/mf6/__init__.py b/flopy4/mf6/__init__.py index 5e65c1d2..553b8b9a 100644 --- a/flopy4/mf6/__init__.py +++ b/flopy4/mf6/__init__.py @@ -1,6 +1,8 @@ +from importlib import import_module from json import dump as dump_json from json import load as load_json from pathlib import Path +from typing import TYPE_CHECKING from tomli import load as load_toml from tomli_w import dump as dump_toml @@ -11,8 +13,10 @@ DFN_SCHEMA_VERSION = "unknown" MF6_VERSION = "unknown" -# Import submodules to make them accessible via flopy4.mf6.* -from flopy4.mf6 import exg, gwe, gwf, gwt, prt, simulation, solution, utils +# Nothing imported eagerly here may import a generated module, so +# `flopy4 mf6 sync` still runs when the generated classes are broken +# or out of date. Generated components load on first use (__getattr__). +from flopy4.mf6 import solution, utils from flopy4.mf6._sync import SyncError, SyncResult, sync from flopy4.mf6._types import TimeArraySeriesRef from flopy4.mf6.codec import dump as dump_mf6 @@ -20,15 +24,58 @@ from flopy4.mf6.component import Component from flopy4.mf6.context import Context from flopy4.mf6.converter import structure, unstructure -from flopy4.mf6.ems import Ems from flopy4.mf6.enums import NetCDFFormat -from flopy4.mf6.ims import Ims from flopy4.mf6.load_context import LoadContext from flopy4.mf6.netcdf import NetCDFModel -from flopy4.mf6.simulation import Simulation -from flopy4.mf6.tdis import Tdis from flopy4.uio import DEFAULT_REGISTRY +if TYPE_CHECKING: + from flopy4.mf6 import ems as ems + from flopy4.mf6 import exg, gwe, gwf, gwt, prt, simulation + from flopy4.mf6 import ims as ims + from flopy4.mf6 import tdis as tdis + from flopy4.mf6 import utl as utl + from flopy4.mf6.ems import Ems + from flopy4.mf6.ims import Ims + from flopy4.mf6.simulation import Simulation + from flopy4.mf6.tdis import Tdis + +# Modules holding the generated components. Importing them all +# registers every component class (see FNAMES in component.py). +_COMPONENT_MODULES = ("exg", "gwe", "gwf", "gwt", "prt", "utl", "simulation", "ems", "ims", "tdis") + +# Lazy attributes: name -> (module, attribute or None for the module). +_LAZY = { + **{m: (m, None) for m in _COMPONENT_MODULES}, + "Ems": ("ems", "Ems"), + "Ims": ("ims", "Ims"), + "Simulation": ("simulation", "Simulation"), + "Tdis": ("tdis", "Tdis"), +} + + +def _import_components() -> None: + """Import every generated component module.""" + for name in _COMPONENT_MODULES: + import_module(f"{__name__}.{name}") + + +def __getattr__(name: str): + if name not in _LAZY: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + _import_components() + module, attr = _LAZY[name] + value = import_module(f"{__name__}.{module}") + if attr is not None: + value = getattr(value, attr) + globals()[name] = value + return value + + +def __dir__(): + return sorted(set(globals()) | set(_LAZY)) + + __all__ = [ "exg", "gwf", diff --git a/flopy4/mf6/_sync.py b/flopy4/mf6/_sync.py index 24823dd2..aeca2d6e 100644 --- a/flopy4/mf6/_sync.py +++ b/flopy4/mf6/_sync.py @@ -43,15 +43,11 @@ class SyncResult: The directory the classes and contract were written to. files : The generated module files. - kept : - Generated modules from an earlier sync that this source has no - DFN for. They're left in place, since other modules may import - them, and stay in the contract's file list. installed : The MF6 executable sync installed, if asked to. removed : - Generated modules that neither this sync nor the previous - contract lists, e.g. ones an earlier sync created before + Generated modules this sync didn't regenerate: ones whose DFN + the source lacks, or ones an earlier sync created before ``pip install --upgrade`` restored the release's contract. """ @@ -60,7 +56,6 @@ class SyncResult: source: str outdir: Path files: tuple[Path, ...] - kept: tuple[Path, ...] = () removed: tuple[Path, ...] = () installed: Path | None = None @@ -326,8 +321,9 @@ def sync( """Regenerate the ``flopy4.mf6`` classes for an MF6 version. Fetches or reads the DFNs, regenerates the component classes, and - records the MF6 version in ``_contract.py``. Installs the matching - MF6 binary only if ``install`` is set. + records the MF6 version in ``_contract.py``. Generated modules the + DFNs don't cover are removed. Installs the matching MF6 binary only + if ``install`` is set. Parameters ---------- @@ -486,22 +482,8 @@ def sync( except SyntaxError as e: raise SyncError(f"Generated invalid Python for {rel}: {e}") from e - on_disk = _generated_files(outdir) - listed = _contract_files(outdir) - if listed is None: - kept = on_disk - generated - removed: set[str] = set() - else: - kept = (on_disk & listed) - generated - removed = on_disk - listed - generated - if kept: - warnings.warn( - f"{len(kept)} generated module(s) have no DFN in '{source}' and were " - f"left from the previous sync: {', '.join(sorted(kept))}", - stacklevel=2, - ) - - (stage / _CONTRACT).write_text(_contract_text(version, commit, generated | kept)) + removed = _generated_files(outdir) - generated + (stage / _CONTRACT).write_text(_contract_text(version, commit, generated)) _swap( stage, outdir, @@ -517,7 +499,6 @@ def sync( source=str(source), outdir=outdir, files=tuple(spec.outpath for spec in specs), - kept=tuple(outdir / f for f in sorted(kept)), removed=tuple(outdir / f for f in sorted(removed)), installed=installed, ) diff --git a/flopy4/mf6/component.py b/flopy4/mf6/component.py index 96458719..9eb6de1c 100644 --- a/flopy4/mf6/component.py +++ b/flopy4/mf6/component.py @@ -34,12 +34,18 @@ def _qualify(name: str, prefix: "str | None") -> str: def get_fnames() -> "dict[str, type[Component]]": """Get a map of MF6 component name (e.g. 'gwf-dis') to component class.""" + from flopy4.mf6 import _import_components + + _import_components() return FNAMES def get_ftypes() -> "dict[str, type[Component]]": """Get a map of MF6 component ftype (e.g. 'gwf-dis6') to component class.""" if not FTYPES: + from flopy4.mf6 import _import_components + + _import_components() from collections import defaultdict from flopy4.mf6.converter.binding import component_ftype diff --git a/flopy4/mf6/utils/codegen/make.py b/flopy4/mf6/utils/codegen/make.py index 452062a1..100834db 100644 --- a/flopy4/mf6/utils/codegen/make.py +++ b/flopy4/mf6/utils/codegen/make.py @@ -10,7 +10,6 @@ schema this replaces. """ -import importlib.util from collections.abc import Mapping from dataclasses import dataclass from dataclasses import field as dc_field @@ -1117,12 +1116,15 @@ def _has_class(target: str, link: File, dfns: Mapping[str, Component] | None) -> return _has_module(target) +# The installed flopy4.mf6 package. +_MF6_ROOT = Path(__file__).parents[2] + + def _has_module(target: str) -> bool: - """Whether a component's generated module exists.""" - try: - return importlib.util.find_spec(_component_module(target)) is not None - except ModuleNotFoundError: # no subpackage either (chf-dis) - return False + """Whether a component's generated module exists in the installed + package. Checks the file rather than importing, so a sync works + when the generated classes don't import.""" + return filters.output_path(target, _MF6_ROOT).is_file() def _child_link( diff --git a/test/mf6/test_mf6_sync.py b/test/mf6/test_mf6_sync.py index 84213c9f..356d8a52 100644 --- a/test/mf6/test_mf6_sync.py +++ b/test/mf6/test_mf6_sync.py @@ -35,6 +35,57 @@ def test_exported(): assert flopy4.mf6.SyncError is SyncError +# Run with every generated component module unimportable. +_BROKEN = """ +import sys + +class Broken: + def find_spec(self, name, path, target=None): + parts = name.split(".") + if parts[:2] == ["flopy4", "mf6"] and len(parts) > 2 and parts[2] in {mods!r}: + raise ImportError(f"broken: {{name}}") + +sys.meta_path.insert(0, Broken()) +""" + + +def test_import_skips_generated(): + """Importing flopy4, the CLI and sync loads no generated module.""" + from flopy4.mf6 import _COMPONENT_MODULES + + code = ( + "import sys, flopy4, flopy4.cli, flopy4.mf6._sync\n" + f"mods = {set(_COMPONENT_MODULES)!r}\n" + "print([m for m in sys.modules if m.split('.')[:2] == ['flopy4', 'mf6']" + " and len(m.split('.')) > 2 and m.split('.')[2] in mods])" + ) + out = subprocess.check_output([sys.executable, "-W", "ignore", "-c", code], text=True) + assert out.strip() == "[]" + + +def test_sync_with_broken_classes(dfn_path, tmp_path): + """Sync, and status, run when the generated classes don't import.""" + from flopy4.mf6 import _COMPONENT_MODULES + + outdir = tmp_path / "mf6" + code = _BROKEN.format(mods=set(_COMPONENT_MODULES)) + ( + "from flopy4 import cli\n" + "from flopy4.mf6 import sync\n" + f"sync({str(dfn_path)!r}, mf6_version='6.9.0', all_packages=True, outdir={str(outdir)!r})\n" + "sys.argv = ['flopy4', 'mf6', 'status']\n" + "cli.main()\n" + "try:\n" + " import flopy4.mf6\n" + " flopy4.mf6.Simulation\n" + "except ImportError:\n" + " pass\n" + "else:\n" + " raise AssertionError('generated classes imported')\n" + ) + subprocess.run([sys.executable, "-W", "ignore", "-c", code], check=True) + assert (outdir / "gwf" / "__init__.py").is_file() + + def test_sync_local_all_packages(dfn_path, tmp_path): """A local sync generates every component and writes the contract.""" result = sync(dfn_path, mf6_version="6.9.0", all_packages=True, outdir=tmp_path) @@ -116,7 +167,7 @@ def test_contract_lists_files(outdir): def test_resync_is_stable(dfn_path, outdir): before = _snapshot(outdir) result = sync(dfn_path, mf6_version="6.9.0", outdir=outdir) - assert result.kept == result.removed == () + assert result.removed == () assert _snapshot(outdir) == before @@ -131,30 +182,37 @@ def test_orphan_removed(dfn_path, outdir): assert find_orphans(outdir) == [] -def test_listed_without_dfn_kept(dfn_path, outdir): - """A listed module with no DFN stays: other modules may import it.""" +def test_listed_without_dfn_removed(dfn_path, outdir): + """A module from an earlier sync whose DFN the source lacks goes.""" stale = outdir / "gwf" / "zzz.py" stale.write_text(_HEADER) contract = outdir / "_contract.py" contract.write_text(contract.read_text().replace("(\n", '(\n "gwf/zzz.py",\n', 1)) - with pytest.warns(UserWarning, match="gwf/zzz.py"): - result = sync(dfn_path, mf6_version="6.9.0", outdir=outdir) - assert result.kept == (stale,) - assert stale.exists() - assert "gwf/zzz.py" in _read_contract(outdir)["GENERATED_FILES"] + assert find_orphans(outdir) == [] + result = sync(dfn_path, mf6_version="6.9.0", outdir=outdir) + assert result.removed == (stale,) + assert not stale.exists() + assert "gwf/zzz.py" not in _read_contract(outdir)["GENERATED_FILES"] -def test_no_file_list_keeps_everything(dfn_path, outdir): - """A contract from before sync tracked files: nothing is removed.""" +def test_no_file_list_removes_stale(dfn_path, outdir): + """A contract from before sync tracked files: the header still marks + a module as generated, so it goes.""" stale = outdir / "gwf" / "zzz.py" stale.write_text(_HEADER) (outdir / "_contract.py").write_text(_HEADER + 'MF6_VERSION = "develop"\n') assert find_orphans(outdir) == [] - with pytest.warns(UserWarning, match="gwf/zzz.py"): - result = sync(dfn_path, mf6_version="6.9.0", outdir=outdir) - assert result.kept == (stale,) - assert result.removed == () - assert stale.exists() + result = sync(dfn_path, mf6_version="6.9.0", outdir=outdir) + assert result.removed == (stale,) + assert not stale.exists() + + +def test_handwritten_module_survives(dfn_path, outdir): + """Modules without the generated header are never removed.""" + mine = outdir / "gwf" / "mine.py" + mine.write_text("x = 1\n") + sync(dfn_path, mf6_version="6.9.0", outdir=outdir) + assert mine.read_text() == "x = 1\n" def test_failed_swap_rolls_back(dfn_path, outdir, monkeypatch):