🔧 Advanced Import Paths and Module Discovery

Master the mechanics of module discovery and implement custom import systems.

🎯 sys.path Manipulation Strategies

Advanced sys.path techniques for complex project structures.

import sys
import os
from pathlib import Path

class ImportPathManager:
    """Manage sys.path for complex projects."""

    def __init__(self):
        self.original_path = sys.path.copy()

    def add_project_root(self, marker_file=".projectroot"):
        """Find project root and add to path."""
        current = Path.cwd()

        # Search up the tree for marker file
        while current != current.parent:
            if (current / marker_file).exists():
                if str(current) not in sys.path:
                    sys.path.insert(0, str(current))
                return current

            current = current.parent

        raise RuntimeError("Could not find project root")

    def add_relative_paths(self, *relative_paths):
        """Add multiple relative paths."""
        base = Path(__file__).parent.parent

        for rel_path in relative_paths:
            full_path = str(base / rel_path)
            if full_path not in sys.path:
                sys.path.insert(0, full_path)

    def setup_development(self):
        """Setup for development mode."""
        # Add src/, tests/, lib/ to path
        self.add_relative_paths("src", "tests", "lib")

        # Add plugin directories
        plugin_dir = Path.cwd() / "plugins"
        if plugin_dir.exists():
            sys.path.append(str(plugin_dir))

    def setup_production(self):
        """Setup for production mode."""
        # Only add necessary paths
        self.add_relative_paths("src")

        # Remove relative paths
        sys.path = [p for p in sys.path if not p.startswith(".")]

    def restore(self):
        """Restore original path."""
        sys.path[:] = self.original_path

# Usage
manager = ImportPathManager()
manager.setup_development()

💡 Namespace Directories and Discovery

import sys
import importlib.util
from pathlib import Path

class NamespacePackageFinder:
    """Find and manage namespace packages."""

    def __init__(self, namespace_name, search_paths=None):
        self.namespace = namespace_name
        self.search_paths = search_paths or sys.path
        self.locations = []

    def discover_locations(self):
        """Find all locations contributing to namespace."""
        for search_path in self.search_paths:
            namespace_dir = Path(search_path) / self.namespace
            if namespace_dir.is_dir() and not (namespace_dir / "__init__.py").exists():
                # This is a namespace package contribution
                self.locations.append(namespace_dir)

        return self.locations

    def get_modules(self):
        """Get all modules available in namespace."""
        modules = {}

        for location in self.locations:
            for py_file in location.glob("*.py"):
                if not py_file.name.startswith("_"):
                    module_name = py_file.stem
                    if module_name not in modules:
                        modules[module_name] = []
                    modules[module_name].append(py_file)

        return modules

    def load_all_modules(self):
        """Dynamically load all modules in namespace."""
        loaded = {}

        for module_name in self.get_modules():
            try:
                full_name = f"{self.namespace}.{module_name}"
                loaded[module_name] = importlib.import_module(full_name)
            except ImportError as e:
                print(f"Failed to load {module_name}: {e}")

        return loaded

# Usage
finder = NamespacePackageFinder("plugins", ["/path1", "/path2"])
finder.discover_locations()
modules = finder.load_all_modules()

🏗️ Custom Import Hooks with sys.meta_path

import sys
import importlib.abc
import importlib.machinery
from pathlib import Path

class ModuleInterceptor(importlib.abc.MetaPathFinder):
    """Intercept and track all imports."""

    def __init__(self):
        self.imported = []
        self.import_times = {}

    def find_spec(self, fullname, path, target=None):
        """Track import."""
        import time

        self.imported.append(fullname)
        self.import_times[fullname] = time.time()

        # Let other finders handle actual import
        return None

    def report(self):
        """Report import statistics."""
        print(f"Imported {len(self.imported)} modules")
        print(f"Import times: {self.import_times}")

class CachingFinder(importlib.abc.MetaPathFinder):
    """Cache module specs for faster imports."""

    def __init__(self):
        self.cache = {}

    def find_spec(self, fullname, path, target=None):
        """Return cached spec if available."""
        if fullname in self.cache:
            return self.cache[fullname]

        return None

    def cache_spec(self, fullname, spec):
        """Store spec in cache."""
        self.cache[fullname] = spec

# Usage
interceptor = ModuleInterceptor()
sys.meta_path.insert(0, interceptor)

# After imports
interceptor.report()

📦 Path-Based Module Loading

import importlib.util
import sys
from pathlib import Path

class PathBasedLoader:
    """Load modules from specific paths."""

    @staticmethod
    def load_from_file(module_name, file_path):
        """Load module from file path."""
        file_path = Path(file_path)

        spec = importlib.util.spec_from_file_location(
            module_name,
            file_path
        )

        if spec and spec.loader:
            module = importlib.util.module_from_spec(spec)
            sys.modules[module_name] = module
            spec.loader.exec_module(module)
            return module

        raise ImportError(f"Could not load {module_name} from {file_path}")

    @staticmethod
    def load_from_directory(package_name, dir_path):
        """Load package from directory."""
        dir_path = Path(dir_path)
        init_file = dir_path / "__init__.py"

        if not init_file.exists():
            raise ImportError(f"Package {package_name} missing __init__.py")

        return PathBasedLoader.load_from_file(package_name, init_file)

    @staticmethod
    def load_submodules(package_name, dir_path):
        """Load all submodules from directory."""
        dir_path = Path(dir_path)
        modules = {}

        for py_file in dir_path.glob("*.py"):
            if not py_file.name.startswith("_"):
                module_name = f"{package_name}.{py_file.stem}"
                modules[py_file.stem] = PathBasedLoader.load_from_file(
                    module_name, py_file
                )

        return modules

# Usage
# module = PathBasedLoader.load_from_file("mymodule", "/path/to/module.py")
# package = PathBasedLoader.load_from_directory("mypackage", "/path/to/package")

🔄 Dynamic sys.path Configuration

import sys
import os
from contextlib import contextmanager

class SysPathContext:
    """Context manager for temporary sys.path modifications."""

    def __init__(self, *new_paths):
        self.new_paths = new_paths
        self.old_paths = None

    def __enter__(self):
        """Save and modify sys.path."""
        self.old_paths = sys.path.copy()

        for path in self.new_paths:
            if path not in sys.path:
                sys.path.insert(0, path)

        return self

    def __exit__(self, *args):
        """Restore sys.path."""
        sys.path[:] = self.old_paths

@contextmanager
def temporary_import_path(*paths):
    """Context manager for temporary import paths."""
    old_path = sys.path.copy()

    try:
        for path in paths:
            if path not in sys.path:
                sys.path.insert(0, path)
        yield
    finally:
        sys.path[:] = old_path

# Usage
with temporary_import_path("/custom/path"):
    import custom_module  # Available in this context

# custom_module no longer importable

💻 Module Discovery and Analysis

import sys
import importlib.util
from pathlib import Path

class ModuleDiscovery:
    """Discover and analyze available modules."""

    @staticmethod
    def find_modules(search_paths=None):
        """Find all modules in search paths."""
        search_paths = search_paths or sys.path
        modules = set()

        for search_path in search_paths:
            path = Path(search_path)
            if not path.exists():
                continue

            # Find .py files
            for py_file in path.rglob("*.py"):
                if py_file.name != "__init__.py":
                    relative = py_file.relative_to(path)
                    module_name = str(relative).replace("/", ".")[:-3]
                    modules.add(module_name)

        return sorted(modules)

    @staticmethod
    def analyze_module(module_name):
        """Analyze module properties."""
        try:
            spec = importlib.util.find_spec(module_name)

            if spec is None:
                return None

            return {
                "name": module_name,
                "origin": spec.origin,
                "is_package": spec.submodule_search_locations is not None,
                "loader": spec.loader.__class__.__name__,
                "parent": spec.parent,
            }
        except (ImportError, ValueError):
            return None

    @staticmethod
    def get_dependency_tree(module_name, max_depth=3):
        """Get import dependency tree."""
        import ast

        tree = {module_name: {}}
        _build_tree(module_name, tree[module_name], 0, max_depth)
        return tree

# Helper for tree building
def _build_tree(module_name, node, depth, max_depth):
    """Recursively build dependency tree."""
    if depth >= max_depth:
        return

    try:
        spec = importlib.util.find_spec(module_name)
        if spec and spec.origin:
            with open(spec.origin) as f:
                tree = ast.parse(f.read())

            for import_node in ast.walk(tree):
                if isinstance(import_node, ast.Import):
                    for alias in import_node.names:
                        node[alias.name] = {}
    except Exception:
        pass

🔑 Key Takeaways

• ✅ Manage sys.path for complex project structures

• ✅ Use context managers for temporary path modifications

• ✅ Implement custom finders for special import behavior

• ✅ Cache module specs for performance

• ✅ Discover modules dynamically from directories

• ✅ Analyze module dependencies programmatically

• ✅ Handle namespace packages with path discovery

Ready to practice? Challenges | Quiz