Skip to content
79 changes: 65 additions & 14 deletions IPython/core/magic.py
Original file line number Diff line number Diff line change
Expand Up @@ -397,10 +397,13 @@ def __init__(self, manager: MagicsManager) -> None:
self._manager = manager

def __missing__(self, key: str) -> Any:
for magic_name, spec in list(self._manager.lazy_magics.items()):
for magic_name, magic_kind, spec in self._manager._lazy_declarations():
if spec.endswith(":" + key):
self._manager.load_lazy(magic_name)
break
self._manager.load_lazy(magic_name, magic_kind)
# A matching spec declared once per kind
# has one spec per kind, so keep going until `key` shows up.
if key in self:
break
if key not in self:
# A second miss must not loop back here.
raise KeyError(key)
Expand Down Expand Up @@ -569,16 +572,26 @@ def register_lazy(
continue
self.magics[kind][name] = LazyMagic(self, fully_qualified_name, kind, name)

def load_lazy(self, magic_name: str) -> None:
def load_lazy(self, magic_name: str, magic_kind: _MagicKind | None = None) -> None:
"""Import and register whatever provides `magic_name`.

Does nothing if `magic_name` was not declared through
:meth:`register_lazy` or :attr:`lazy_magics`, or if what provides it
has already been loaded.

When `magic_kind` is given, the placeholder for that kind decides
which spec to load: a name may be declared with a different provider
per kind, and the lookup must resolve the kind that was asked for,
not whichever placeholder happens to be checked first.
"""
# `lazy_magics` is user-configurable and may have been replaced
# wholesale, so prefer the spec the placeholder carries.
fn = self.magics["line"].get(magic_name) or self.magics["cell"].get(magic_name)
if magic_kind is not None:
fn = self.magics[magic_kind].get(magic_name)

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch here!

else:
fn = self.magics["line"].get(magic_name) or self.magics["cell"].get(
magic_name
)
spec = (
fn.spec if isinstance(fn, LazyMagic) else self.lazy_magics.get(magic_name)
)
Expand All @@ -604,31 +617,69 @@ def load_lazy(self, magic_name: str) -> None:
self._loaded_lazy.discard(spec)
raise

def _lazy_declarations(self) -> list[tuple[str, _MagicKind | None, str]]:
"""Every live lazy declaration, as ``(name, kind, spec)``.

The placeholders in :attr:`magics` are the source of truth, because
:attr:`lazy_magics` is keyed by name alone and so keeps only the last
spec for a name declared once per kind. A name declared only through
the trait has no placeholder, and so no known kind.
"""
seen: set[tuple[str, str]] = set()
declarations: list[tuple[str, _MagicKind | None, str]] = []
for kind in magic_kinds:
for name, fn in list(self.magics[kind].items()):
if isinstance(fn, LazyMagic) and (name, fn.spec) not in seen:
seen.add((name, fn.spec))
declarations.append((name, kind, fn.spec))
for name, spec in list(self.lazy_magics.items()):
if (name, spec) not in seen:
seen.add((name, spec))
declarations.append((name, None, spec))
return declarations

def load_all_lazy_magics(self) -> None:
"""Import and register every magic still declared lazily.

Only the ``module:MagicsClass`` ones: loading an extension can run
arbitrary code, so that waits for the magic to actually be used.
"""
for magic_name, spec in list(self.lazy_magics.items()):
for magic_name, magic_kind, spec in self._lazy_declarations():
if ":" in spec:
self.load_lazy(magic_name)
self.load_lazy(magic_name, magic_kind)

def find(
self, magic_kind: _MagicKind, magic_name: str
) -> Callable[..., Any] | None:
"""Return a registered magic, importing its implementation if needed.

Returns None if there is no such magic.

Raises
------
UsageError
If `magic_name` is declared lazily for `magic_kind` but what the
declaration names does not provide it.
"""
fn = self.magics[magic_kind].get(magic_name)
if isinstance(fn, LazyMagic) or (fn is None and magic_name in self.lazy_magics):
self.load_lazy(magic_name)
fn = self.magics[magic_kind].get(magic_name)
table = self.magics[magic_kind]
fn = table.get(magic_name)
if fn is None and magic_name in self.lazy_magics:
# Declared straight into `lazy_magics` as configuration, so there
# is no placeholder for this kind to carry the spec.
self.load_lazy(magic_name, magic_kind)
fn = table.get(magic_name)
if isinstance(fn, LazyMagic):
self.load_lazy(magic_name, magic_kind)
fn = table.get(magic_name)
if isinstance(fn, LazyMagic):
# Declared but not delivered; drop the stale placeholder.
del self.magics[magic_kind][magic_name]
fn = None
# Declared but not delivered: the provider does not
# implement this kind, and that is an error rather than a
# cue to fall back to an older declaration.
raise UsageError(
"Magic `%s%s` is registered as lazy (%s) but loading it "
"did not provide a %s implementation."
% (magic_escapes[magic_kind], magic_name, fn.spec, magic_kind)
)
return t.cast("Callable[..., Any] | None", fn)

def register(self, *magic_objects: type[Magics] | Magics) -> None:
Expand Down
234 changes: 234 additions & 0 deletions tests/test_magic.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,15 @@

import pytest

from traitlets.config import Config, Configurable

from IPython import get_ipython
from IPython.core import magic
from IPython.core.error import UsageError
from IPython.core.interactiveshell import InteractiveShellABC
from IPython.core.magic import (
Magics,
MagicsManager,
cell_magic,
line_magic,
magics_class,
Expand Down Expand Up @@ -62,6 +66,22 @@ class DummyMagics(magic.Magics):
pass


class _UnconfiguredShell(Configurable):
"""Just enough of a shell for a ``Magics`` class to be instantiated."""

def __init__(self):
super().__init__(config=Config())


InteractiveShellABC.register(_UnconfiguredShell)


@pytest.fixture
def standalone_manager():
"""A standalone MagicsManager, so tests don't disturb the shared shell."""
return MagicsManager(shell=_UnconfiguredShell())


def test_extract_code_ranges():
instr = "1 3 5-6 7-9 10:15 17: :10 10- -13 :"
expected = [
Expand Down Expand Up @@ -1978,6 +1998,220 @@ def edit(self, line):
mm.registry.pop("OverridingMagics", None)


SPLIT_LAZY_MAGIC = """
from IPython.core.magic import Magics, cell_magic, line_magic, magics_class


@magics_class
class LineHalf(Magics):
@line_magic
def dual(self, line):
\"\"\"The line half of the dual magic.\"\"\"


@magics_class
class CellHalf(Magics):
@cell_magic
def dual(self, line, cell):
\"\"\"The cell half of the dual magic.\"\"\"
"""


@pytest.mark.parametrize("line_first", [True, False])
def test_lazy_magic_help_finds_both_kinds_first_try(line_first):
"""`?` help resolves a lazily declared magic on the first attempt.

The same name may be declared with a different provider per kind; the
lookup must load the provider for the kind that was asked for, no
matter in which order the declarations were registered.
See https://github.com/ipython/ipython/issues/15383.
"""
mm = ip.magics_manager
with TemporaryDirectory() as tmpdir:
with prepended_to_syspath(tmpdir):
mod = "split_lazy_magic_module"
Path(tmpdir, mod + ".py").write_text(dedent(SPLIT_LAZY_MAGIC))
invalidate_caches()
line_spec = f"{mod}:LineHalf"
cell_spec = f"{mod}:CellHalf"
first, second = (
(("line", line_spec), ("cell", cell_spec))
if line_first
else (("cell", cell_spec), ("line", line_spec))
)
try:
mm.register_lazy("dual", first[1], first[0]) # type: ignore[arg-type]

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
mm.register_lazy("dual", first[1], first[0]) # type: ignore[arg-type]
mm.register_lazy("dual", first[1], first[0])

mm.register_lazy("dual", second[1], second[0]) # type: ignore[arg-type]

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
mm.register_lazy("dual", second[1], second[0]) # type: ignore[arg-type]
mm.register_lazy("dual", second[1], second[0])

# Both kinds resolve on the very first lookup ...
assert "dual" in mm.magics["cell"]
assert mm.find("cell", "dual") is not None
assert not isinstance(mm.find("cell", "dual"), magic.LazyMagic)
assert mm.find("line", "dual") is not None
assert not isinstance(mm.find("line", "dual"), magic.LazyMagic)
# ... and so does `?` help in both spellings.
assert ip._ofind("%%dual").found
assert ip._ofind("%dual").found
finally:
mm.magics["line"].pop("dual", None)
mm.magics["cell"].pop("dual", None)
mm.lazy_magics.pop("dual", None)
mm._loaded_lazy.discard(line_spec)
mm._loaded_lazy.discard(cell_spec)
mm.registry.pop("LineHalf", None)
mm.registry.pop("CellHalf", None)
sys.modules.pop(mod, None)


def test_lazy_magic_missing_implementation_raises(standalone_manager):
"""A declaration that delivers nothing is an error, not a fallback.

If a lazily declared class only provides the line half of a magic,
asking for the cell half raises instead of silently falling back to
a previous provider.
See https://github.com/ipython/ipython/issues/15383.
"""
mm = standalone_manager
with TemporaryDirectory() as tmpdir:
with prepended_to_syspath(tmpdir):
mod = "line_only_lazy_magic_module"
Path(tmpdir, mod + ".py").write_text(
dedent(
"""
from IPython.core.magic import Magics, line_magic, magics_class


@magics_class
class LineOnly(Magics):
@line_magic
def time(self, line):
\"\"\"My own line-only time.\"\"\"
"""
)
)
invalidate_caches()
spec = f"{mod}:LineOnly"
try:
mm.register_lazy("time", spec)
assert mm.find("line", "time") is not None
with pytest.raises(UsageError):
mm.find("cell", "time")
finally:
mm.magics["cell"].pop("time", None)
mm.magics["line"].pop("time", None)
mm.lazy_magics.pop("time", None)
mm._loaded_lazy.discard(spec)
mm.registry.pop("LineOnly", None)
sys.modules.pop(mod, None)


def test_lazy_magic_missing_implementation_raises_at_any_depth():
"""Stacked declarations that deliver nothing still raise.

Three lazy declarations for one name where none provides ``mydepth``:
the lookup raises instead of giving up quietly or returning nothing.
See https://github.com/ipython/ipython/issues/15383.
"""
mm = ip.magics_manager
with TemporaryDirectory() as tmpdir:
with prepended_to_syspath(tmpdir):
Path(tmpdir, "depth_first_mod.py").write_text(
dedent(
"""
from IPython.core.magic import Magics, line_magic, magics_class


@magics_class
class First(Magics):
@line_magic
def mydepth(self, line):
\"\"\"The original provider.\"\"\"
"""
)
)
for mod in ("depth_second_mod", "depth_third_mod"):
Path(tmpdir, mod + ".py").write_text(
dedent(
"""
from IPython.core.magic import Magics, magics_class


@magics_class
class Empty(Magics):
pass
"""
)
)
invalidate_caches()
specs = [
"depth_first_mod:First",
"depth_second_mod:Empty",
"depth_third_mod:Empty",
]
try:
for spec in specs:
mm.register_lazy("mydepth", spec, "line")
with pytest.raises(UsageError):
mm.find("line", "mydepth")
finally:
mm.magics["line"].pop("mydepth", None)
mm.lazy_magics.pop("mydepth", None)
for spec in specs:
mm._loaded_lazy.discard(spec)
mm.registry.pop("First", None)
mm.registry.pop("Empty", None)
for mod in ("depth_first_mod", "depth_second_mod", "depth_third_mod"):
sys.modules.pop(mod, None)


def test_load_all_lazy_magics_loads_both_kind_halves(standalone_manager):
"""`load_all_lazy_magics` loads every kind's provider, not just one.

One name may be declared with a different provider per kind, but
``lazy_magics`` only remembers the last spec per name — so loading from
it alone (line-first) never imports the other half's class, and e.g.
``%config CellHalf.trait`` fails until something else loads it.
See https://github.com/ipython/ipython/issues/15383.
"""
mm = standalone_manager
with TemporaryDirectory() as tmpdir:
with prepended_to_syspath(tmpdir):
mod = "split_load_all_module"
Path(tmpdir, mod + ".py").write_text(
dedent(
"""
from IPython.core.magic import Magics, line_magic, cell_magic, magics_class


@magics_class
class LineHalf(Magics):
@line_magic
def dual(self, line):
\"\"\"The line half.\"\"\"


@magics_class
class CellHalf(Magics):
@cell_magic
def dual(self, line, cell):
\"\"\"The cell half.\"\"\"
"""
)
)
invalidate_caches()
line_spec = f"{mod}:LineHalf"
cell_spec = f"{mod}:CellHalf"
try:
mm.register_lazy("dual", line_spec, "line") # type: ignore[arg-type]

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
mm.register_lazy("dual", line_spec, "line") # type: ignore[arg-type]
mm.register_lazy("dual", line_spec, "line")

mm.register_lazy("dual", cell_spec, "cell") # type: ignore[arg-type]

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
mm.register_lazy("dual", cell_spec, "cell") # type: ignore[arg-type]
mm.register_lazy("dual", cell_spec, "cell")

mm.load_all_lazy_magics()

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Calling load_all_lazy_magics() on the shared session shell may break later tests.
Lets build a standalone MagicsManager(shell=...) for this test instead of using the session ip. There's a manager fixture in test_magic_table.py

assert "LineHalf" in mm.registry
assert "CellHalf" in mm.registry
assert not isinstance(mm.find("line", "dual"), magic.LazyMagic)
assert not isinstance(mm.find("cell", "dual"), magic.LazyMagic)
finally:
sys.modules.pop(mod, None)


TEST_MODULE = """
print('Loaded my_tmp')
if __name__ == "__main__":
Expand Down
Loading
Loading