hyperproc.atmos.setup

Source-derived reference

Generated from the current hyperproc 0.1.2 checkout.

Implementation: hyperproc/atmos/setup.py. Signatures, defaults, docstrings, and expandable source are extracted statically; the module is not imported or executed. Names beginning with _ are implementation details, not a stable public API.

Use the function signature as the authority for individual parameter defaults and return annotations. Original docstrings sometimes group parameter names or wrap return descriptions across lines; these descriptions are preserved rather than inferred or rewritten.

One-time environment setup and health check for :mod:hyperproc.atmos.

ISOFIT is a pip package, but the radiative-transfer engines it drives are not: 6S is Fortran that must be compiled on the machine, sRTMnet is a 5.5 GB set of emulator weights, LibRadTran is a compiled C/Fortran package, and the data and surface libraries add another ~100 MB. ISOFIT keeps all of them under one base directory named in ~/.isofit/isofit.ini. On a shared machine point every user at one base (setup(base="/data/.../isofit_assets")) so the assets are fetched once.

• :func:check reports what is installed and what is missing. It never downloads anything.
• :func:setup fetches (and compiles) what the requested engines need, then runs :func:check.
• hyperproc-atmos-setup is the command-line form of :func:setup.

ENGINES = ('sRTMnet', '6s', 'LibRadTran') module-attribute

NEEDS = {'sRTMnet': ('data', 'surface', 'sixs', 'srtmnet'), '6s': ('data', 'surface', 'sixs'), 'LibRadTran': ('data', 'surface', 'libradtran')} module-attribute

COMPILERS = {'sixs': ('gfortran', 'make'), 'libradtran': ('gcc', 'gfortran', 'make', 'gsl-config')} module-attribute

PIP_HINT = "pip install 'hyperproc[atmos]' (installs isofit>=4.1, torch and ray)" module-attribute

_quiet(*_a, **_k)

Source code in hyperproc/atmos/setup.py

def _quiet(*_a, **_k):
    pass

_isofit()

Source code in hyperproc/atmos/setup.py

def _isofit():
    try:
        import isofit
    except ImportError:
        return None
    return isofit

_assets_for(engines)

Source code in hyperproc/atmos/setup.py

def _assets_for(engines):
    bad = [e for e in engines if e not in ENGINES]
    if bad:
        raise ValueError(f"unknown engine(s) {bad}; choose from {ENGINES}")
    return sorted({k for e in engines for k in NEEDS[e]})

libradtran_exe(path) -> Path | None

bin/uvspec under a libRadtran asset directory, or None when unbuilt.

Source code in hyperproc/atmos/setup.py

def libradtran_exe(path) -> Path | None:
    """``bin/uvspec`` under a libRadtran asset directory, or None when unbuilt."""
    base = Path(path)
    for cand in [base] + sorted(p for p in base.glob("*") if p.is_dir()):
        exe = cand / "bin" / "uvspec"
        if exe.is_file():
            return exe
    return None

build_libradtran(path, jobs: int = 8, verbose: bool = True) -> Path

Configure and compile an unpacked libRadtran tree.

libRadtran needs the GNU Scientific Library and netCDF at build time. ISOFIT's own build step runs ./configure && make without checking either, and silently leaves no binary when GSL is missing. This builds against the libraries of the running conda environment (sys.prefix; conda install -c conda-forge gsl supplies GSL) and verifies that bin/uvspec exists afterwards.

Source code in hyperproc/atmos/setup.py

def build_libradtran(path, jobs: int = 8, verbose: bool = True) -> Path:
    """Configure and compile an unpacked libRadtran tree.

    libRadtran needs the GNU Scientific Library and netCDF at build time.
    ISOFIT's own build step runs ``./configure && make`` without checking
    either, and silently leaves no binary when GSL is missing. This builds
    against the libraries of the running conda environment (``sys.prefix``;
    ``conda install -c conda-forge gsl`` supplies GSL) and verifies that
    ``bin/uvspec`` exists afterwards.
    """
    import os
    import subprocess

    base = Path(path)
    trees = [p for p in [base] + sorted(base.glob("*")) if (p / "configure").is_file()]
    if not trees:
        raise FileNotFoundError(f"no libRadtran source tree (configure script) under {base}; run isofit download libradtran")
    tree = trees[0]
    prefix = Path(sys.prefix)
    env = dict(os.environ)
    env["PATH"] = f"{prefix / 'bin'}:{env.get('PATH', '')}"
    env["CPPFLAGS"] = f"-I{prefix / 'include'} {env.get('CPPFLAGS', '')}".strip()
    env["LDFLAGS"] = f"-L{prefix / 'lib'} -Wl,-rpath,{prefix / 'lib'} {env.get('LDFLAGS', '')}".strip()
    # The tarball ships pre-generated dependency lists (src/.depend) that name
    # the developers' /usr/include/gsl headers; make then fails looking for
    # them. They are regenerated by the build, so drop the shipped copies.
    for dep in tree.rglob(".depend"):
        try:
            if "/usr/include" in dep.read_text(errors="replace"):
                dep.unlink()
        except OSError:
            pass
    log = tree / "hyperproc_build.log"
    with open(log, "w") as fh:
        for cmd in (["./configure", "--with-netcdf4=yes"], ["make", f"-j{int(jobs)}"]):
            if verbose:
                print(f"libradtran: {' '.join(cmd)}  (log {log})", flush=True)
            proc = subprocess.run(cmd, cwd=str(tree), env=env, stdout=fh, stderr=subprocess.STDOUT)
            if proc.returncode != 0:
                tail = "\n".join(log.read_text(errors="replace").splitlines()[-15:])
                raise RuntimeError(f"libRadtran {cmd[0]} failed (exit {proc.returncode}); tail of {log}:\n{tail}")
    exe = libradtran_exe(base)
    if exe is None:
        raise RuntimeError(f"libRadtran built without error but bin/uvspec is missing under {tree}; see {log}")
    return exe

check(engines=('sRTMnet',), verbose: bool = True) -> dict

What the atmospheric-correction stack has and lacks, for engines.

Returns a dict with isofit (version or None), ini, base, compilers, assets (per ISOFIT key: path, ok, exe) and missing (human-readable items with the command that supplies each). ok is True when nothing is missing. Nothing is downloaded or written.

Source code in hyperproc/atmos/setup.py

def check(engines=("sRTMnet",), verbose: bool = True) -> dict:
    """What the atmospheric-correction stack has and lacks, for ``engines``.

    Returns a dict with ``isofit`` (version or None), ``ini``, ``base``,
    ``compilers``, ``assets`` (per ISOFIT key: path, ok, exe) and ``missing``
    (human-readable items with the command that supplies each). ``ok`` is
    True when nothing is missing. Nothing is downloaded or written.
    """
    report = {"python": sys.version.split()[0], "isofit": None, "ini": None, "base": None,
              "compilers": {}, "assets": {}, "missing": [], "ok": False}
    iso = _isofit()
    if iso is None:
        report["missing"].append(f"isofit: {PIP_HINT}")
        _report(report, verbose)
        return report
    report["isofit"] = iso.__version__
    from isofit.data import env

    report["ini"] = str(env.ini)
    report["base"] = str(getattr(env, "base", "")) or None
    for rc in unterminated_rc_files():
        report["missing"].append(
            f"{rc}: printf '\\n' >> {rc}   (no newline at the end; netCDF-C "
            f"reads past it and ISOFIT's workers crash at random)")
    needed_tools = sorted({t for k in _assets_for(engines) for t in COMPILERS.get(k, ())})
    for tool in needed_tools:
        # the conda environment's own bin (gsl-config lives there) counts even when it is not on PATH
        report["compilers"][tool] = shutil.which(tool) or shutil.which(tool, path=str(Path(sys.prefix) / "bin"))
        if report["compilers"][tool] is None:
            pkg = "gsl" if tool == "gsl-config" else tool
            report["missing"].append(f"{tool}: conda install -c conda-forge {pkg}   (needed to build the engine)")
    for key in _assets_for(engines):
        path = env[key]
        entry = {"path": path, "ok": False}
        try:
            mod = importlib.import_module(f"isofit.data.cli.{key}")
            entry["ok"] = bool(mod.validate(path, checkForUpdate=False, debug=_quiet, error=_quiet))
        except Exception as exc:  # a broken install must show as missing, not crash the check
            entry["error"] = f"{type(exc).__name__}: {exc}"
        if entry["ok"] and key == "sixs":
            from isofit.data.cli.sixs import get_exe
            entry["exe"] = str(get_exe(path))
        if entry["ok"] and key == "libradtran":
            # ISOFIT's validator accepts an unpacked, unbuilt tree; the engine needs bin/uvspec
            exe = libradtran_exe(path)
            if exe is None:
                entry["ok"] = False
                entry["error"] = "source unpacked but not built (no bin/uvspec)"
                report["missing"].append(f"{key}: build failed or not run; hyperproc-atmos-setup --engine LibRadTran "
                                         f"builds it against the conda GSL/netCDF (needs gcc, gfortran, make, gsl)")
                report["assets"][key] = entry
                continue
            entry["exe"] = str(exe)
        if not entry["ok"]:
            report["missing"].append(f"{key}: isofit download {key}   -> {path}")
        report["assets"][key] = entry
    report["ok"] = not report["missing"]
    _report(report, verbose)
    return report

_report(report: dict, verbose: bool) -> None

Source code in hyperproc/atmos/setup.py

def _report(report: dict, verbose: bool) -> None:
    if not verbose:
        return
    print(f"hyperproc.atmos check  (python {report['python']}, isofit {report['isofit'] or 'not installed'})")
    if report["ini"]:
        print(f"  ini      {report['ini']}")
    for tool, where in report["compilers"].items():
        print(f"  {tool:8s} {where or 'MISSING'}")
    for key, e in report["assets"].items():
        state = "ok" if e["ok"] else "MISSING"
        extra = f"  exe {e['exe']}" if e.get("exe") else (f"  ({e['error']})" if e.get("error") else "")
        print(f"  {key:10s} {state:8s} {e['path']}{extra}")
    if report["missing"]:
        print("  to fix:")
        for m in report["missing"]:
            print(f"    - {m}")
    else:
        print("  everything needed is in place")

setup(base: str | None = None, engines=('sRTMnet',), examples: bool = False, overwrite: bool = False, verbose: bool = True) -> dict

Fetch and build what engines need, then :func:check.

PARAMETER	DESCRIPTION
base	directory to hold every ISOFIT asset. Recorded in ~/.isofit/isofit.ini so ISOFIT and every later call find it; None keeps the current ini setting (default ~/.isofit). TYPE: str | None DEFAULT: None
engines	which engines to provision. "sRTMnet" (default) is the emulator JPL uses operationally and needs 6S compiled underneath. DEFAULT: ('sRTMnet',)
examples	also download ISOFIT's tutorial data (~340 MB), useful for an end-to-end smoke test of a fresh install. TYPE: bool DEFAULT: False
overwrite	re-download assets that validate fine. TYPE: bool DEFAULT: False

RAISES	DESCRIPTION
ImportError	ISOFIT is not installed.
RuntimeError	a compiler an engine needs is missing.

Source code in hyperproc/atmos/setup.py

def setup(base: str | None = None, engines=("sRTMnet",), examples: bool = False,
          overwrite: bool = False, verbose: bool = True) -> dict:
    """Fetch and build what ``engines`` need, then :func:`check`.

    Args:
        base: directory to hold every ISOFIT asset. Recorded in
            ``~/.isofit/isofit.ini`` so ISOFIT and every later call find it;
            None keeps the current ini setting (default ``~/.isofit``).
        engines: which engines to provision. ``"sRTMnet"`` (default) is the
            emulator JPL uses operationally and needs 6S compiled underneath.
        examples: also download ISOFIT's tutorial data (~340 MB), useful for
            an end-to-end smoke test of a fresh install.
        overwrite: re-download assets that validate fine.

    Raises:
        ImportError: ISOFIT is not installed.
        RuntimeError: a compiler an engine needs is missing.
    """
    iso = _isofit()
    if iso is None:
        raise ImportError(f"ISOFIT is not installed: {PIP_HINT}")
    from isofit.data import env

    if base:
        Path(base).expanduser().mkdir(parents=True, exist_ok=True)
        env.changeBase(str(Path(base).expanduser().resolve()))
        env.save(diff_only=False)
        if verbose:
            print(f"ISOFIT assets base -> {env.base}  (saved to {env.ini})")
    for rc in unterminated_rc_files():
        if terminate_rc_file(rc) and verbose:
            print(f"{rc}: added the missing newline at the end of the file "
                  f"(netCDF-C reads past a last line without one)")
    keys = _assets_for(engines)
    missing_tools = [t for k in keys for t in COMPILERS.get(k, ())
                     if shutil.which(t) is None and shutil.which(t, path=str(Path(sys.prefix) / "bin")) is None]
    if missing_tools:
        raise RuntimeError(f"cannot build the engines without {sorted(set(missing_tools))}; "
                           f"install them first (conda install -c conda-forge {' '.join(sorted(set(missing_tools)))})")
    for key in keys:
        mod = importlib.import_module(f"isofit.data.cli.{key}")
        if not overwrite and mod.validate(env[key], checkForUpdate=False, debug=_quiet, error=_quiet):
            if verbose:
                print(f"{key}: already present at {env[key]}")
            continue
        if key == "libradtran" and not overwrite and Path(env[key]).exists() and libradtran_exe(env[key]) is None:
            if verbose:
                print(f"{key}: source present but not built -> compiling")
            build_libradtran(env[key], verbose=verbose)
            continue
        if verbose:
            print(f"{key}: downloading{' and building' if key in COMPILERS else ''} -> {env[key]}")
        mod.download(overwrite=overwrite)
        if key == "libradtran" and libradtran_exe(env[key]) is None:
            build_libradtran(env[key], verbose=verbose)
    if examples:
        from isofit.data.cli import examples as ex
        ex.download(overwrite=overwrite)
    return check(engines, verbose=verbose)

main(argv=None) -> int

hyperproc-atmos-setup: provision the ISOFIT stack, or just check it.

Source code in hyperproc/atmos/setup.py

def main(argv=None) -> int:
    """``hyperproc-atmos-setup``: provision the ISOFIT stack, or just check it."""
    import argparse

    p = argparse.ArgumentParser(prog="hyperproc-atmos-setup",
                                description="Fetch and build the ISOFIT engines and assets hyperproc.atmos needs.")
    p.add_argument("--base", help="directory for all ISOFIT assets; default ~/.isofit. On a shared machine give every user the same one so the assets are fetched once")
    p.add_argument("--engine", action="append", choices=ENGINES, help="engine(s) to provision; default sRTMnet")
    p.add_argument("--examples", action="store_true", help="also fetch ISOFIT's tutorial data")
    p.add_argument("--overwrite", action="store_true", help="re-download assets that already validate")
    p.add_argument("--check", action="store_true", help="only report what is present; download nothing")
    a = p.parse_args(argv)
    engines = tuple(a.engine) if a.engine else ("sRTMnet",)
    try:
        report = check(engines) if a.check else setup(a.base, engines, a.examples, a.overwrite)
    except (ImportError, RuntimeError, ValueError) as exc:
        print(f"error: {exc}", file=sys.stderr)
        return 2
    return 0 if report["ok"] else 1