🌐 Advanced Namespace Packages and Distribution

Design sophisticated distributed packages using PEP 420 namespace packages.

🎯 PEP 420 Implicit Namespace Packages

Deep dive into namespace package mechanics.

import sys
import types

# Check if a package is a namespace package
def is_namespace_package(module_name):
    """Determine if module is a namespace package."""
    if module_name not in sys.modules:
        return None

    module = sys.modules[module_name]

    # Namespace packages have __path__ but no __file__
    return (
        hasattr(module, "__path__") and
        not hasattr(module, "__file__")
    )

# Examine namespace package properties
import myapp  # (namespace package without __init__.py)

print("__path__:", myapp.__path__)          # List of paths
print("__file__:", getattr(myapp, "__file__", "Not set"))  # None
print("__spec__:", myapp.__spec__)          # ModuleSpec

💡 Multi-Location Namespace Architecture

# Location 1: /usr/lib/company/
company/
├── core/
│   ├── utils.py
│   └── models.py
(no __init__.py!)

# Location 2: /opt/company/
company/
├── plugins/
│   ├── plugin_a.py
│   └── plugin_b.py
(no __init__.py!)

# Location 3: /home/user/company/
company/
└── custom/
    └── extension.py
(no __init__.py!)

# All three contribute to single "company" namespace!

Implementation:

import sys
from pathlib import Path

def setup_namespace_paths():
    """Setup namespace package paths."""
    locations = [
        "/usr/lib/company",
        "/opt/company",
        "/home/user/company"
    ]

    for loc in locations:
        if loc not in sys.path:
            sys.path.insert(0, loc)

setup_namespace_paths()

# Now all modules are accessible
from company.core import utils
from company.plugins import plugin_a
from company.custom import extension

print(utils.__file__)        # /usr/lib/company/core/utils.py
print(plugin_a.__file__)     # /opt/company/plugins/plugin_a.py
print(extension.__file__)    # /home/user/company/custom/extension.py

🏗️ Namespace Package with Entry Points

# setup.py - Configure namespace packages
from setuptools import setup, find_namespace_packages

setup(
    name="company-core",
    version="1.0.0",
    packages=find_namespace_packages(),
    # Find packages like company.* without __init__.py
)

# setup.py - Another team's package
setup(
    name="company-plugins",
    version="1.0.0",
    packages=find_namespace_packages(),
    entry_points={
        "company.plugins": [
            "myplugin = my_module:MyPlugin",
        ]
    },
)

# At runtime, both contribute to "company" namespace

📦 Plugin Discovery in Namespace Packages

import sys
import importlib
from pathlib import Path
from typing import Dict, Type

class NamespacePluginLoader:
    """Load plugins from namespace packages."""

    def __init__(self, namespace: str, submodule: str):
        self.namespace = namespace
        self.submodule = submodule
        self.plugins: Dict[str, Type] = {}

    def discover(self):
        """Discover all plugins in namespace."""
        # Get all paths for namespace package
        namespace_module = importlib.import_module(self.namespace)

        for base_path in namespace_module.__path__:
            submodule_path = Path(base_path) / self.submodule

            if not submodule_path.exists():
                continue

            # Find all modules in submodule
            for py_file in submodule_path.glob("*.py"):
                if py_file.name.startswith("_"):
                    continue

                module_name = (
                    f"{self.namespace}.{self.submodule}."
                    f"{py_file.stem}"
                )

                try:
                    module = importlib.import_module(module_name)
                    self.plugins[py_file.stem] = module
                except ImportError as e:
                    print(f"Failed to load {module_name}: {e}")

        return self.plugins

    def get_plugin_classes(self, base_class):
        """Get all plugin classes inheriting from base."""
        plugin_classes = {}

        for name, module in self.plugins.items():
            for item_name in dir(module):
                item = getattr(module, item_name)
                if (isinstance(item, type) and
                    issubclass(item, base_class) and
                    item is not base_class):
                    plugin_classes[name] = item

        return plugin_classes

# Usage
loader = NamespacePluginLoader("company", "plugins")
loader.discover()

from company.plugins.base import Plugin
plugin_classes = loader.get_plugin_classes(Plugin)

🔄 Merging Namespace Packages

import sys
import importlib
from pathlib import Path
from types import ModuleType

class NamespaceMerger:
    """Merge multiple namespace package locations."""

    @staticmethod
    def merge_modules(namespace: str):
        """Merge all modules from namespace locations."""
        ns_module = importlib.import_module(namespace)

        # Collect all modules from all locations
        all_modules = {}

        for base_path in ns_module.__path__:
            base_path = Path(base_path)

            for py_file in base_path.rglob("*.py"):
                if py_file.name == "__init__.py":
                    continue

                relative = py_file.relative_to(base_path)
                module_name = (
                    f"{namespace}."
                    f"{str(relative)[:-3].replace('/', '.')}"
                )

                # Import all
                try:
                    if module_name not in sys.modules:
                        importlib.import_module(module_name)
                except ImportError:
                    pass

        return all_modules

    @staticmethod
    def get_all_submodules(namespace: str):
        """Get all submodules in namespace."""
        NamespaceMerger.merge_modules(namespace)

        namespace_module = importlib.import_module(namespace)
        return [
            name for name, obj in vars(namespace_module).items()
            if isinstance(obj, ModuleType)
        ]

💻 Namespace Package with Configuration

# company/__init__.py (namespace package - no file!)
# Instead, use a configuration loader

# config_loader.py
import importlib
import sys
from pathlib import Path

class NamespaceConfig:
    """Configuration for namespace package."""

    def __init__(self, namespace: str):
        self.namespace = namespace
        self._config = {}

    def load_from_locations(self):
        """Load configuration from all namespace locations."""
        ns_module = importlib.import_module(self.namespace)

        for base_path in ns_module.__path__:
            config_file = Path(base_path) / "config.toml"
            if config_file.exists():
                # Load TOML or JSON configuration
                import json
                with open(config_file) as f:
                    self._config.update(json.load(f))

        return self._config

    def get(self, key, default=None):
        """Get configuration value."""
        return self._config.get(key, default)

# Usage
config = NamespaceConfig("company")
config.load_from_locations()
debug_mode = config.get("debug", False)

🎨 Vendor Namespace Packages

Real-world example inspired by zope.* pattern.

# Multiple independent vendors in same namespace
zope/
├── interface/         # Zope Interface (no __init__.py!)
│   ├── interface.py
│   └── verify.py
zope/                  # Different location
├── component/         # Zope Component (no __init__.py!)
│   ├── adapter.py
│   └── registry.py
zope/                  # Yet another location
└── configuration/     # Zope Configuration (no __init__.py!)
    ├── config.py
    └── loader.py

Setup and usage:

import sys

# Add all vendor locations
sys.path.extend([
    "/vendor/zope-interface",
    "/vendor/zope-component",
    "/vendor/zope-configuration"
])

# All accessible through "zope" namespace!
from zope.interface import Interface
from zope.component import adapter
from zope.configuration import config

# Each can be independently versioned and updated

📊 Namespace vs Regular Package Comparison

⚠️ Common Namespace Issues

Issue 1: Mixing regular and namespace packages

# ❌ Problem: confusing package structure
zope/
├── __init__.py          # Regular package!
├── interface/
│   └── interface.py     # Namespace component

✅ Solution: Use only namespace packages

# Correct: all use namespace pattern
zope/
├── interface/           # No __init__.py
│   └── interface.py
zope/
└── component/           # No __init__.py
    └── component.py

Issue 2: Import order matters

import sys

# Must add paths BEFORE importing namespace
sys.path.insert(0, "/location1")
sys.path.insert(0, "/location2")

from mypackage.modules import something  # Works

# This would fail:
from mypackage.modules import something
sys.path.insert(0, "/location3")  # Too late!

🔑 Key Takeaways

• ✅ Namespace packages (PEP 420) don't require __init__.py

• ✅ Multiple locations can contribute to same namespace

• ✅ Ideal for plugin systems and vendor packages

• ✅ Use importlib to discover modules dynamically

• ✅ All paths must be in sys.path before import

• ✅ Can't store package metadata without __init__.py

• ✅ More complex but powerful for distributed systems

Ready to practice? Challenges | Quiz