Source code for phenotypic.sdk_

"""Developer tools shared across fungal colony plate workflows.

Lightweight helpers for timing, mask validation, constants, color conversions, error
handling, and HDF storage used by the processing pipeline. Includes a timed execution
decorator, mask validators, colourspace utilities, custom exceptions, and HDF helpers
for persisting plate datasets and measurements.

Advanced users can access GridInferenceMixin and FootprintMixin for creating custom
grid-based operations and morphological footprints.

The ``_io_constants`` submodule is the single source of truth for CLI ↔ GUI
artifact filenames, directory names, JSON contract keys, and path helpers
(re-exported here at package level for convenience).
"""

import importlib as _importlib
from typing import TYPE_CHECKING as _TYPE_CHECKING
from typing import Any as _Any

#: Re-exports whose submodules import heavy third-party libraries -- colour-science
#: (``colourspace``), h5py (``hdf_``), pandas (``_measurement_tables``,
#: ``_metadata_migration``), scipy and scikit-image (``mixin``). Importing any
#: ``phenotypic.sdk_.*`` module runs this ``__init__`` first, so these resolve on first
#: access instead. Defined before the eager imports below, so an import that re-enters
#: this package mid-initialisation still resolves them.
_LAZY_ATTRS: dict[str, str] = {
    "colourspace": ".colourspace",
    "HDF": ".hdf_",
    "PreparedEmbeddedMeasurementTable": "._measurement_tables",
    "PreparedImageTables": "._measurement_tables",
    "build_measurement_table_descriptor": "._measurement_tables",
    "build_metadata_table_descriptor": "._measurement_tables",
    "embedded_measurement_columns": "._measurement_tables",
    "read_embedded_measurement_column": "._measurement_tables",
    "read_embedded_measurement_descriptor": "._measurement_tables",
    "replace_embedded_measurement_table": "._measurement_tables",
    "replace_image_tables": "._measurement_tables",
    "write_embedded_measurement_table": "._measurement_tables",
    "write_image_tables": "._measurement_tables",
    "write_metadata_table": "._measurement_tables",
    "MetadataMigrationAuthority": "._metadata_migration",
    "MetadataMigrationReport": "._metadata_migration",
    "MetadataMigrationResult": "._metadata_migration",
    "MetadataMigrationTarget": "._metadata_migration",
    "metadata_migration_authority": "._metadata_migration",
    "migrate_metadata_bundle": "._metadata_migration",
    "migrate_metadata_file": "._metadata_migration",
    "migrate_preflighted_metadata_bundle": "._metadata_migration",
    "preflight_metadata_schema": "._metadata_migration",
    "rollback_metadata_migration": "._metadata_migration",
    "validated_published_metadata_migration_targets": "._metadata_migration",
    "FootprintMixin": ".mixin",
    "GridInferenceMixin": ".mixin",
    "InputLayerMixin": ".mixin",
    "LazyWidgetMixin": ".mixin",
    "NormControlMixin": ".mixin",
    "NormalizedOutputMixin": ".mixin",
}


[docs] def __getattr__(name: str) -> _Any: """Resolve a heavy re-export on first access and cache it on the package.""" module_name = _LAZY_ATTRS.get(name) if module_name is None: raise AttributeError(f"module {__name__!r} has no attribute {name!r}") module = _importlib.import_module(module_name, __name__) value = module if module_name == f".{name}" else getattr(module, name) globals()[name] = value return value
def __dir__() -> list[str]: # ``globals().get`` rather than a bare ``__all__``: this function sits above the eager # imports for the same reason ``__getattr__`` does, so it is callable in a window where # ``__all__`` has not been assigned yet. A bare reference raises ``NameError`` there. return sorted(set(globals()) | set(globals().get("__all__", ()))) if _TYPE_CHECKING: from . import colourspace from ._measurement_tables import ( PreparedEmbeddedMeasurementTable, PreparedImageTables, build_measurement_table_descriptor, build_metadata_table_descriptor, embedded_measurement_columns, read_embedded_measurement_column, read_embedded_measurement_descriptor, replace_embedded_measurement_table, replace_image_tables, write_embedded_measurement_table, write_image_tables, write_metadata_table, ) from ._metadata_migration import ( MetadataMigrationAuthority, MetadataMigrationReport, MetadataMigrationResult, MetadataMigrationTarget, metadata_migration_authority, migrate_metadata_bundle, migrate_metadata_file, migrate_preflighted_metadata_bundle, preflight_metadata_schema, rollback_metadata_migration, validated_published_metadata_migration_targets, ) from .hdf_ import HDF from .mixin import ( FootprintMixin, GridInferenceMixin, InputLayerMixin, LazyWidgetMixin, NormControlMixin, NormalizedOutputMixin, ) from . import ( # noqa: E402 constants_, exceptions_, napari_, slurm, slurm_, ) from . import _io_constants # noqa: E402 from ._atomic_io import ( # noqa: E402 CommitGuard, PARQUET_WRITE_OPTIONS, atomic_write_bytes, atomic_write_json, atomic_write_parquet, atomic_write_text, atomic_write_with_writer, publication_commit, ) from ._column_ref import ColumnRef, ColumnRefList, ColumnSource # noqa: E402 from ._io_constants import ( # noqa: E402 # Filenames (CLI artifacts) CURATION_LABELS_PARQUET, CUSTOM_CATEGORIES_JSON, DIR_ERRORS, ERROR_ANALYSIS_CSV, ERROR_ANALYSIS_HTML, ERROR_ANALYSIS_PARQUET, ANALYSIS_MANIFEST_FILENAME, ANALYSIS_MANIFEST_SCHEMA_VERSION, ANALYSIS_FULL_PARQUET, BEST_PARAMS_JSON, BEST_PIPELINE_JSON, CHUNK_LOCK, CHUNK_MANIFEST_JSON, CHUNK_STATE_JSON, CONFIG_SUFFIX_COLOR_CHECKER, CONFIG_SUFFIX_OPERATION, CONFIG_SUFFIX_PIPELINE, CONFIG_SUFFIX_TUNING, CONFIG_SUFFIXES, DASHBOARD_HTML, DATASET_AGGREGATED_PARQUET, DELIVERABLES_METADATA_CSV, DIR_AGGREGATION_SHARDS, DIR_CHUNKS, DIR_DELIVERABLES, DIR_IMAGE_COMPLETE, DIR_IMAGE_RECORDS, DIR_LEGACY_V2, DIR_LOGS, DIR_MEASUREMENTS, DIR_MEASUREMENTS_BY_FEATURE, DIR_OVERLAYS, DIR_PLOTS, DIR_PARETO, DIR_PHENOTYPIC, DIR_PHT_TUNE_CACHE, DIR_PROGRESS, DIR_QC, DIR_RECOMPILE, DIR_RECOMPILE_SHARDS, DIR_RECOMPILE_STATUS, DIR_RESULTS, DIR_SLURM_SCRIPTS, DIR_SPLITS, DIR_STAGE2_DONE, DIR_ZARR, MIGRATION_REMEDY, FAILURES_JSONL, TERMINAL_FAILURES_JSONL, GENERALIZATION_JSON, GUI_LAUNCH_OWNER_JSON, GUI_LOG_FILENAMES, JOB_METADATA_JSON, LEGACY_JSON_SUFFIX, LOCAL_SCHEDULER_EPOCH, MANIFEST_JSON, MASTER_MEASUREMENTS_PARQUET, MEASUREMENTS_CSV, MEASUREMENTS_PARQUET, PARAM_IMPORTANCE_JSON, PARETO_FRONT_PARQUET, PIPELINE_JSON, PIPELINE_CONFIG_SUFFIXES, PROCESSING_EVENTS_LOG, PROCESSING_REPORT_HTML, PROCESSING_STATE_JSON, AGGREGATE_PROOF_VERSION, ARTIFACT_KIND_FILE, ARTIFACT_KIND_STORE, RUN_COMPLETION_JSON, RUN_LOG_DIRNAME, RUN_MARKER_JSON, RUN_PROOF_VERSION, SLURM_LIFECYCLE_JSON, SUCCESS_MARKER_VERSION, QC_DUCKDB, QC_REVIEW_STATE_JSON, README_MD, RECOMPILE_TASK_MANIFEST_JSON, RESTART_EPOCH_JSON, REMBI_MANIFEST_YAML, SENTINEL_RESUBMITTED_MARKER, SPLIT_ASSIGNMENT_JSON, STDOUT_LOG, STUDY_DB, STUDY_JOURNAL_LOG, TRIALS_PARQUET, TUNING_CONFIG_SUFFIXES, TUNING_SPEC_JSON, VERIFICATION_CACHE_JSON, VERIFICATION_CACHE_VERSION, VERIFIED_PARQUET, # Namespace classes ChunkManifestKey, ChunkStateKey, DashboardManifestKey, DashboardManifestSlurmInfoKey, EnvVar, JobMetadataKey, ModulePath, ProcessingStateKey, AnalysisArtifactPaths, # Path helpers analysis_full_parquet_path, aggregate_publication_marker_path, analysis_manifest_path, best_pipeline_path, best_params_path, bytes_fingerprint, chunk_lock_path, chunk_manifest_path, chunk_parquet_filename, chunk_parquet_path, chunk_state_path, chunks_dir, aggregation_shard_dir, checkpoint_lock_filename, checkpoint_lock_path, dashboard_html_path, BundleLayout, dataset_measurements_dir, dataset_overlays_dir, dataset_results_dir, dataset_zarr_dir, datasets_needing_migration, overlays_dir, plots_dir, default_output_dir_name, curation_labels_parquet_path, custom_categories_json_path, deliverables_dir, error_analysis_csv_path, error_analysis_html_path, error_analysis_parquet_path, error_category_parquet_path, errors_dir, event_log_path, ensure_typed_json_suffix, failures_jsonl_path, file_fingerprint, generalization_path, generation_staging_path, gui_launch_owner_path, slurm_lifecycle_path, image_completion_marker_path, image_record_path, has_config_suffix, job_metadata_path, load_image_from_store, logs_dir, manifest_json_path, master_measurements_parquet_path, measurements_by_feature_dir, measurements_csv_path, measurements_parquet_path, migration_backup_dir, migration_backup_path, migration_lock_path, migration_receipt_path, named_analysis_csv_path, named_analysis_parquet_path, named_analysis_paths, metadata_csv_deliverable_path, terminal_failures_jsonl_path, matches_any_suffix, param_importance_path, pareto_best_pipeline_path, pareto_dir, pareto_front_parquet_path, pareto_importance_path, phenotypic_cache_dir, phenotypic_cache_pipeline_json_path, pipeline_json_path, pipeline_publication_lock_path, paths_fingerprint, processing_report_html_path, processing_state_path, progress_dir, qc_dir, qc_duckdb_path, qc_review_state_path, readme_md_path, read_run_manifest, recompile_dir, recompile_status_dir, rembi_manifest_path, resolve_event_log_path, resolve_execution_mode, resolve_manifest_json_path, resolve_pipeline_config_path, resolve_processing_state_path, resolve_progress_dir, resolve_split_assignment_path, resolve_study_db_path, resolve_tuning_spec_path, migrate_legacy_machine_state, migrate_legacy_qc, clear_machine_state, restart_epoch_path, results_dir, run_completion_marker_path, sentinel_resubmitted_path, shard_parquet_filename, slurm_scripts_dir, source_cache_key, is_zarr_store_name, store_stem, source_image_stem, source_image_suffix, store_publication_token, store_revision_identity, task_status_filename, task_status_path, trials_parquet_path, tune_cache_dir, tune_cache_run_marker_path, tune_cache_journal_path, tune_cache_split_assignment_path, tune_cache_splits_dir, tune_cache_study_db_path, tuning_spec_path, verification_cache_path, verified_parquet_path, validate_analysis_id, zarr_store_path, ) from ._image_record import ( # noqa: E402 PROVENANCE_FORWARD, PROVENANCE_MIGRATED, RECORD_VERSION, STAGE_MEASURED, STAGE_STAGE1, STAGE_STAGE2, STAGE_STAGE3, read_image_record, record_provenance, record_rejection, ) from ._master_io import ( # noqa: E402 master_carries_user_metadata, user_metadata_headers, ) from ._pipeline_publication import pipeline_publication_lock # noqa: E402 from ._run_state import ( # noqa: E402 ImageState, RunDiagnostics, RunIdentity, RunState, aggregate_proof_is_current, assert_identity_current, clear_verification_cache, finalization_input_object, resolve_run_state, run_identity, ) from .funcs_ import is_binary_mask, timed_execution # noqa: E402 from ._metadata_helpers import ( # noqa: E402 canonical_metadata_order, ensure_metadata_prefix, external_metadata_preserved_columns, is_metadata_header, metadata_category_for_label, metadata_category_prefixes, metadata_member_for_header, metadata_member_for_label, metadata_only_mask, metadata_owner_for_header, metadata_owner_for_label, normalize_metadata_columns, order_measurement_columns, ) from .ngff_ import ( # noqa: E402 EMBEDDED_MEASUREMENT_PARQUET_METADATA_KEYS, MEASUREMENT_TABLE_RELATIVE_PATH, METADATA_TABLE_RELATIVE_PATH, STORE_ROOT_JSON, STORE_SUFFIX, PhenotypicAttr, describe_durability, durable_writes_enabled, new_part_path, promote_store, read_phenotypic_attributes, sweep_orphan_parts, valid_staged_store, ) from .typing_ import InputLayer, NormOut, ProcessOnlyLayer # noqa: E402 __all__ = [ "aggregate_publication_marker_path", # Mixins / utilities (existing) "ColumnRef", "ColumnRefList", "ColumnSource", "FootprintMixin", "GridInferenceMixin", "HDF", "InputLayerMixin", "LazyWidgetMixin", "NormControlMixin", "NormalizedOutputMixin", "CommitGuard", "atomic_write_bytes", "atomic_write_json", "atomic_write_parquet", "atomic_write_text", "atomic_write_with_writer", "publication_commit", "PARQUET_WRITE_OPTIONS", "PreparedEmbeddedMeasurementTable", "PreparedImageTables", "build_measurement_table_descriptor", "build_metadata_table_descriptor", "embedded_measurement_columns", "read_embedded_measurement_column", "read_embedded_measurement_descriptor", "replace_embedded_measurement_table", "replace_image_tables", "write_embedded_measurement_table", "write_image_tables", "write_metadata_table", "canonical_metadata_order", "colourspace", "constants_", "exceptions_", "ensure_metadata_prefix", "external_metadata_preserved_columns", "is_binary_mask", "is_metadata_header", "metadata_category_for_label", "metadata_category_prefixes", "metadata_member_for_header", "metadata_member_for_label", "metadata_only_mask", "metadata_owner_for_header", "metadata_owner_for_label", "normalize_metadata_columns", "order_measurement_columns", "MetadataMigrationAuthority", "MetadataMigrationReport", "MetadataMigrationResult", "MetadataMigrationTarget", "metadata_migration_authority", "migrate_metadata_bundle", "migrate_metadata_file", "migrate_preflighted_metadata_bundle", "preflight_metadata_schema", "rollback_metadata_migration", "validated_published_metadata_migration_targets", "napari_", "slurm", "slurm_", "timed_execution", # OME-Zarr per-image store (NGFF 0.5 / Zarr format v3) "PhenotypicAttr", "EMBEDDED_MEASUREMENT_PARQUET_METADATA_KEYS", "MEASUREMENT_TABLE_RELATIVE_PATH", "METADATA_TABLE_RELATIVE_PATH", "STORE_ROOT_JSON", "STORE_SUFFIX", "describe_durability", "durable_writes_enabled", "new_part_path", "promote_store", "read_phenotypic_attributes", "sweep_orphan_parts", "valid_staged_store", # Run-state readers (spec §5.2). The four frozen types land in P1 # Task 2, clear_verification_cache in Task 3, the identity readers in # Task 4 and resolve_run_state in Task 5 -- each exported by the task # that implements it, so the tree imports cleanly at every commit. "ImageState", "RunDiagnostics", "RunIdentity", "RunState", # P6 Task 2: the GUI's `core_readable` needs the aggregate-proof half of # the question, which was private. Its other half is the CLI's # `state_requires_success_markers`, so `core_readable` itself cannot live # here -- INV-LAYER keeps `_cli` out of `sdk_`. "aggregate_proof_is_current", "assert_identity_current", "clear_verification_cache", "finalization_input_object", "resolve_run_state", "run_identity", # Typing aliases (closed value sets) "InputLayer", "NormOut", "ProcessOnlyLayer", # I/O constants module (also re-exported below) "_io_constants", # Filename constants "ANALYSIS_FULL_PARQUET", "ANALYSIS_MANIFEST_FILENAME", "ANALYSIS_MANIFEST_SCHEMA_VERSION", "CURATION_LABELS_PARQUET", "CUSTOM_CATEGORIES_JSON", "ERROR_ANALYSIS_CSV", "ERROR_ANALYSIS_HTML", "ERROR_ANALYSIS_PARQUET", "BEST_PARAMS_JSON", "BEST_PIPELINE_JSON", "CHUNK_LOCK", "CHUNK_MANIFEST_JSON", "CHUNK_STATE_JSON", "CONFIG_SUFFIX_COLOR_CHECKER", "CONFIG_SUFFIX_OPERATION", "CONFIG_SUFFIX_PIPELINE", "CONFIG_SUFFIX_TUNING", "CONFIG_SUFFIXES", "DASHBOARD_HTML", "DATASET_AGGREGATED_PARQUET", "DELIVERABLES_METADATA_CSV", "FAILURES_JSONL", "TERMINAL_FAILURES_JSONL", "GENERALIZATION_JSON", "GUI_LAUNCH_OWNER_JSON", "GUI_LOG_FILENAMES", "JOB_METADATA_JSON", "LEGACY_JSON_SUFFIX", "LOCAL_SCHEDULER_EPOCH", "MANIFEST_JSON", "MASTER_MEASUREMENTS_PARQUET", "MEASUREMENTS_CSV", "MEASUREMENTS_PARQUET", "PARAM_IMPORTANCE_JSON", "PARETO_FRONT_PARQUET", "PIPELINE_JSON", "PIPELINE_CONFIG_SUFFIXES", "PROCESSING_EVENTS_LOG", "PROCESSING_REPORT_HTML", "PROCESSING_STATE_JSON", "AGGREGATE_PROOF_VERSION", "ARTIFACT_KIND_FILE", "ARTIFACT_KIND_STORE", "RUN_COMPLETION_JSON", "RUN_LOG_DIRNAME", "RUN_MARKER_JSON", "RUN_PROOF_VERSION", "SLURM_LIFECYCLE_JSON", "SUCCESS_MARKER_VERSION", "QC_DUCKDB", "QC_REVIEW_STATE_JSON", "README_MD", "RECOMPILE_TASK_MANIFEST_JSON", "RESTART_EPOCH_JSON", "REMBI_MANIFEST_YAML", "SENTINEL_RESUBMITTED_MARKER", "SPLIT_ASSIGNMENT_JSON", "STDOUT_LOG", "STUDY_DB", "STUDY_JOURNAL_LOG", "TRIALS_PARQUET", "TUNING_CONFIG_SUFFIXES", "TUNING_SPEC_JSON", "VERIFICATION_CACHE_JSON", "VERIFICATION_CACHE_VERSION", "VERIFIED_PARQUET", # Directory constants "DIR_AGGREGATION_SHARDS", "DIR_CHUNKS", "DIR_DELIVERABLES", "DIR_ERRORS", "DIR_IMAGE_COMPLETE", "DIR_IMAGE_RECORDS", "DIR_LEGACY_V2", "DIR_LOGS", "DIR_MEASUREMENTS", "DIR_MEASUREMENTS_BY_FEATURE", "DIR_OVERLAYS", "DIR_PLOTS", "DIR_PARETO", "DIR_PHENOTYPIC", "DIR_PHT_TUNE_CACHE", "DIR_PROGRESS", "DIR_QC", "DIR_RECOMPILE", "DIR_RECOMPILE_SHARDS", "DIR_RECOMPILE_STATUS", "DIR_RESULTS", "DIR_SLURM_SCRIPTS", "DIR_SPLITS", "DIR_STAGE2_DONE", "DIR_ZARR", # Namespace classes for JSON keys + dispatch "ChunkManifestKey", "ChunkStateKey", "DashboardManifestKey", "DashboardManifestSlurmInfoKey", "EnvVar", "JobMetadataKey", "ModulePath", "ProcessingStateKey", "AnalysisArtifactPaths", # Path helpers "BundleLayout", "aggregation_shard_dir", "analysis_full_parquet_path", "analysis_manifest_path", "best_pipeline_path", "best_params_path", "bytes_fingerprint", "chunk_lock_path", "chunk_manifest_path", "chunk_parquet_filename", "chunk_parquet_path", "chunk_state_path", "chunks_dir", "checkpoint_lock_filename", "checkpoint_lock_path", "dashboard_html_path", "dataset_measurements_dir", "dataset_overlays_dir", "dataset_results_dir", "MIGRATION_REMEDY", "dataset_zarr_dir", "datasets_needing_migration", "overlays_dir", "plots_dir", "curation_labels_parquet_path", "custom_categories_json_path", "default_output_dir_name", "deliverables_dir", "error_analysis_csv_path", "error_analysis_html_path", "error_analysis_parquet_path", "error_category_parquet_path", "errors_dir", "event_log_path", "ensure_typed_json_suffix", "failures_jsonl_path", "file_fingerprint", "generalization_path", "generation_staging_path", "gui_launch_owner_path", "slurm_lifecycle_path", "image_completion_marker_path", "image_record_path", "read_image_record", "record_provenance", "record_rejection", "PROVENANCE_FORWARD", "PROVENANCE_MIGRATED", "RECORD_VERSION", "STAGE_MEASURED", "STAGE_STAGE1", "STAGE_STAGE2", "STAGE_STAGE3", "has_config_suffix", "job_metadata_path", "load_image_from_store", "logs_dir", "manifest_json_path", "master_carries_user_metadata", "user_metadata_headers", "master_measurements_parquet_path", "measurements_by_feature_dir", "measurements_csv_path", "measurements_parquet_path", "migration_backup_dir", "migration_backup_path", "migration_lock_path", "migration_receipt_path", "named_analysis_csv_path", "named_analysis_parquet_path", "named_analysis_paths", "metadata_csv_deliverable_path", "terminal_failures_jsonl_path", "matches_any_suffix", "migrate_legacy_machine_state", "migrate_legacy_qc", "clear_machine_state", "param_importance_path", "pareto_best_pipeline_path", "pareto_dir", "pareto_front_parquet_path", "pareto_importance_path", "phenotypic_cache_dir", "phenotypic_cache_pipeline_json_path", "pipeline_json_path", "pipeline_publication_lock", "pipeline_publication_lock_path", "paths_fingerprint", "processing_report_html_path", "processing_state_path", "progress_dir", "qc_dir", "qc_duckdb_path", "qc_review_state_path", "readme_md_path", "read_run_manifest", "recompile_dir", "recompile_status_dir", "rembi_manifest_path", "resolve_event_log_path", "resolve_execution_mode", "resolve_manifest_json_path", "resolve_pipeline_config_path", "resolve_processing_state_path", "resolve_progress_dir", "resolve_split_assignment_path", "resolve_study_db_path", "resolve_tuning_spec_path", "restart_epoch_path", "results_dir", "run_completion_marker_path", "sentinel_resubmitted_path", "shard_parquet_filename", "slurm_scripts_dir", "source_cache_key", "is_zarr_store_name", "store_stem", "source_image_stem", "source_image_suffix", "store_publication_token", "store_revision_identity", "task_status_filename", "task_status_path", "trials_parquet_path", "tune_cache_dir", "tune_cache_run_marker_path", "tune_cache_journal_path", "tune_cache_split_assignment_path", "tune_cache_splits_dir", "tune_cache_study_db_path", "tuning_spec_path", "verification_cache_path", "verified_parquet_path", "validate_analysis_id", "zarr_store_path", ]