509 lines
15 KiB
Python
509 lines
15 KiB
Python
"""Functions to process IPython magics with."""
|
|
|
|
import ast
|
|
import collections
|
|
import dataclasses
|
|
import re
|
|
import secrets
|
|
import sys
|
|
from functools import lru_cache
|
|
from importlib.util import find_spec
|
|
from typing import Optional
|
|
|
|
if sys.version_info >= (3, 10):
|
|
from typing import TypeGuard
|
|
else:
|
|
from typing_extensions import TypeGuard
|
|
|
|
from black.mode import Mode
|
|
from black.output import out
|
|
from black.report import NothingChanged
|
|
|
|
TRANSFORMED_MAGICS = frozenset((
|
|
"get_ipython().run_cell_magic",
|
|
"get_ipython().system",
|
|
"get_ipython().getoutput",
|
|
"get_ipython().run_line_magic",
|
|
))
|
|
TOKENS_TO_IGNORE = frozenset((
|
|
"ENDMARKER",
|
|
"NL",
|
|
"NEWLINE",
|
|
"COMMENT",
|
|
"DEDENT",
|
|
"UNIMPORTANT_WS",
|
|
"ESCAPED_NL",
|
|
))
|
|
PYTHON_CELL_MAGICS = frozenset((
|
|
"capture",
|
|
"prun",
|
|
"pypy",
|
|
"python",
|
|
"python3",
|
|
"time",
|
|
"timeit",
|
|
))
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class Replacement:
|
|
mask: str
|
|
src: str
|
|
|
|
|
|
@lru_cache
|
|
def jupyter_dependencies_are_installed(*, warn: bool) -> bool:
|
|
installed = (
|
|
find_spec("tokenize_rt") is not None and find_spec("IPython") is not None
|
|
)
|
|
if not installed and warn:
|
|
msg = (
|
|
"Skipping .ipynb files as Jupyter dependencies are not installed.\n"
|
|
'You can fix this by running ``pip install "black[jupyter]"``'
|
|
)
|
|
out(msg)
|
|
return installed
|
|
|
|
|
|
def validate_cell(src: str, mode: Mode) -> None:
|
|
"""Check that cell does not already contain TransformerManager transformations,
|
|
or non-Python cell magics, which might cause tokenizer_rt to break because of
|
|
indentations.
|
|
|
|
If a cell contains ``!ls``, then it'll be transformed to
|
|
``get_ipython().system('ls')``. However, if the cell originally contained
|
|
``get_ipython().system('ls')``, then it would get transformed in the same way:
|
|
|
|
>>> TransformerManager().transform_cell("get_ipython().system('ls')")
|
|
"get_ipython().system('ls')\n"
|
|
>>> TransformerManager().transform_cell("!ls")
|
|
"get_ipython().system('ls')\n"
|
|
|
|
Due to the impossibility of safely roundtripping in such situations, cells
|
|
containing transformed magics will be ignored.
|
|
"""
|
|
if any(transformed_magic in src for transformed_magic in TRANSFORMED_MAGICS):
|
|
raise NothingChanged
|
|
|
|
line = _get_code_start(src)
|
|
if line.startswith("%%") and (
|
|
line.split(maxsplit=1)[0][2:]
|
|
not in PYTHON_CELL_MAGICS | mode.python_cell_magics
|
|
):
|
|
raise NothingChanged
|
|
|
|
|
|
def remove_trailing_semicolon(src: str) -> tuple[str, bool]:
|
|
"""Remove trailing semicolon from Jupyter notebook cell.
|
|
|
|
For example,
|
|
|
|
fig, ax = plt.subplots()
|
|
ax.plot(x_data, y_data); # plot data
|
|
|
|
would become
|
|
|
|
fig, ax = plt.subplots()
|
|
ax.plot(x_data, y_data) # plot data
|
|
|
|
Mirrors the logic in `quiet` from `IPython.core.displayhook`, but uses
|
|
``tokenize_rt`` so that round-tripping works fine.
|
|
"""
|
|
from tokenize_rt import reversed_enumerate, src_to_tokens, tokens_to_src
|
|
|
|
tokens = src_to_tokens(src)
|
|
trailing_semicolon = False
|
|
for idx, token in reversed_enumerate(tokens):
|
|
if token.name in TOKENS_TO_IGNORE:
|
|
continue
|
|
if token.name == "OP" and token.src == ";":
|
|
del tokens[idx]
|
|
trailing_semicolon = True
|
|
break
|
|
if not trailing_semicolon:
|
|
return src, False
|
|
return tokens_to_src(tokens), True
|
|
|
|
|
|
def put_trailing_semicolon_back(src: str, has_trailing_semicolon: bool) -> str:
|
|
"""Put trailing semicolon back if cell originally had it.
|
|
|
|
Mirrors the logic in `quiet` from `IPython.core.displayhook`, but uses
|
|
``tokenize_rt`` so that round-tripping works fine.
|
|
"""
|
|
if not has_trailing_semicolon:
|
|
return src
|
|
from tokenize_rt import reversed_enumerate, src_to_tokens, tokens_to_src
|
|
|
|
tokens = src_to_tokens(src)
|
|
for idx, token in reversed_enumerate(tokens):
|
|
if token.name in TOKENS_TO_IGNORE:
|
|
continue
|
|
tokens[idx] = token._replace(src=token.src + ";")
|
|
break
|
|
else: # pragma: nocover
|
|
raise AssertionError(
|
|
"INTERNAL ERROR: Was not able to reinstate trailing semicolon. "
|
|
"Please report a bug on https://github.com/psf/black/issues. "
|
|
) from None
|
|
return str(tokens_to_src(tokens))
|
|
|
|
|
|
def mask_cell(src: str) -> tuple[str, list[Replacement]]:
|
|
"""Mask IPython magics so content becomes parseable Python code.
|
|
|
|
For example,
|
|
|
|
%matplotlib inline
|
|
'foo'
|
|
|
|
becomes
|
|
|
|
b"25716f358c32750"
|
|
'foo'
|
|
|
|
The replacements are returned, along with the transformed code.
|
|
"""
|
|
replacements: list[Replacement] = []
|
|
try:
|
|
ast.parse(src)
|
|
except SyntaxError:
|
|
# Might have IPython magics, will process below.
|
|
pass
|
|
else:
|
|
# Syntax is fine, nothing to mask, early return.
|
|
return src, replacements
|
|
|
|
from IPython.core.inputtransformer2 import TransformerManager
|
|
|
|
transformer_manager = TransformerManager()
|
|
# A side effect of the following transformation is that it also removes any
|
|
# empty lines at the beginning of the cell.
|
|
transformed = transformer_manager.transform_cell(src)
|
|
transformed, cell_magic_replacements = replace_cell_magics(transformed)
|
|
replacements += cell_magic_replacements
|
|
transformed = transformer_manager.transform_cell(transformed)
|
|
transformed, magic_replacements = replace_magics(transformed)
|
|
if len(transformed.strip().splitlines()) != len(src.strip().splitlines()):
|
|
# Multi-line magic, not supported.
|
|
raise NothingChanged
|
|
replacements += magic_replacements
|
|
return transformed, replacements
|
|
|
|
|
|
def create_token(n_chars: int) -> str:
|
|
"""Create a randomly generated token that is n_chars characters long."""
|
|
assert n_chars > 0
|
|
n_bytes = max(n_chars // 2 - 1, 1)
|
|
token = secrets.token_hex(n_bytes)
|
|
if len(token) + 3 > n_chars:
|
|
token = token[:-1]
|
|
# We use a bytestring so that the string does not get interpreted
|
|
# as a docstring.
|
|
return f'b"{token}"'
|
|
|
|
|
|
def get_token(src: str, magic: str) -> str:
|
|
"""Return randomly generated token to mask IPython magic with.
|
|
|
|
For example, if 'magic' was `%matplotlib inline`, then a possible
|
|
token to mask it with would be `"43fdd17f7e5ddc83"`. The token
|
|
will be the same length as the magic, and we make sure that it was
|
|
not already present anywhere else in the cell.
|
|
"""
|
|
assert magic
|
|
n_chars = len(magic)
|
|
token = create_token(n_chars)
|
|
counter = 0
|
|
while token in src:
|
|
token = create_token(n_chars)
|
|
counter += 1
|
|
if counter > 100:
|
|
raise AssertionError(
|
|
"INTERNAL ERROR: Black was not able to replace IPython magic. "
|
|
"Please report a bug on https://github.com/psf/black/issues. "
|
|
f"The magic might be helpful: {magic}"
|
|
) from None
|
|
return token
|
|
|
|
|
|
def replace_cell_magics(src: str) -> tuple[str, list[Replacement]]:
|
|
"""Replace cell magic with token.
|
|
|
|
Note that 'src' will already have been processed by IPython's
|
|
TransformerManager().transform_cell.
|
|
|
|
Example,
|
|
|
|
get_ipython().run_cell_magic('t', '-n1', 'ls =!ls\\n')
|
|
|
|
becomes
|
|
|
|
"a794."
|
|
ls =!ls
|
|
|
|
The replacement, along with the transformed code, is returned.
|
|
"""
|
|
replacements: list[Replacement] = []
|
|
|
|
tree = ast.parse(src)
|
|
|
|
cell_magic_finder = CellMagicFinder()
|
|
cell_magic_finder.visit(tree)
|
|
if cell_magic_finder.cell_magic is None:
|
|
return src, replacements
|
|
header = cell_magic_finder.cell_magic.header
|
|
mask = get_token(src, header)
|
|
replacements.append(Replacement(mask=mask, src=header))
|
|
return f"{mask}\n{cell_magic_finder.cell_magic.body}", replacements
|
|
|
|
|
|
def replace_magics(src: str) -> tuple[str, list[Replacement]]:
|
|
"""Replace magics within body of cell.
|
|
|
|
Note that 'src' will already have been processed by IPython's
|
|
TransformerManager().transform_cell.
|
|
|
|
Example, this
|
|
|
|
get_ipython().run_line_magic('matplotlib', 'inline')
|
|
'foo'
|
|
|
|
becomes
|
|
|
|
"5e67db56d490fd39"
|
|
'foo'
|
|
|
|
The replacement, along with the transformed code, are returned.
|
|
"""
|
|
replacements = []
|
|
magic_finder = MagicFinder()
|
|
magic_finder.visit(ast.parse(src))
|
|
new_srcs = []
|
|
for i, line in enumerate(src.split("\n"), start=1):
|
|
if i in magic_finder.magics:
|
|
offsets_and_magics = magic_finder.magics[i]
|
|
if len(offsets_and_magics) != 1: # pragma: nocover
|
|
raise AssertionError(
|
|
f"Expecting one magic per line, got: {offsets_and_magics}\n"
|
|
"Please report a bug on https://github.com/psf/black/issues."
|
|
)
|
|
col_offset, magic = (
|
|
offsets_and_magics[0].col_offset,
|
|
offsets_and_magics[0].magic,
|
|
)
|
|
mask = get_token(src, magic)
|
|
replacements.append(Replacement(mask=mask, src=magic))
|
|
line = line[:col_offset] + mask
|
|
new_srcs.append(line)
|
|
return "\n".join(new_srcs), replacements
|
|
|
|
|
|
def unmask_cell(src: str, replacements: list[Replacement]) -> str:
|
|
"""Remove replacements from cell.
|
|
|
|
For example
|
|
|
|
"9b20"
|
|
foo = bar
|
|
|
|
becomes
|
|
|
|
%%time
|
|
foo = bar
|
|
"""
|
|
for replacement in replacements:
|
|
src = src.replace(replacement.mask, replacement.src)
|
|
return src
|
|
|
|
|
|
def _get_code_start(src: str) -> str:
|
|
"""Provides the first line where the code starts.
|
|
|
|
Iterates over lines of code until it finds the first line that doesn't
|
|
contain only empty spaces and comments. It removes any empty spaces at the
|
|
start of the line and returns it. If such line doesn't exist, it returns an
|
|
empty string.
|
|
"""
|
|
for match in re.finditer(".+", src):
|
|
line = match.group(0).lstrip()
|
|
if line and not line.startswith("#"):
|
|
return line
|
|
return ""
|
|
|
|
|
|
def _is_ipython_magic(node: ast.expr) -> TypeGuard[ast.Attribute]:
|
|
"""Check if attribute is IPython magic.
|
|
|
|
Note that the source of the abstract syntax tree
|
|
will already have been processed by IPython's
|
|
TransformerManager().transform_cell.
|
|
"""
|
|
return (
|
|
isinstance(node, ast.Attribute)
|
|
and isinstance(node.value, ast.Call)
|
|
and isinstance(node.value.func, ast.Name)
|
|
and node.value.func.id == "get_ipython"
|
|
)
|
|
|
|
|
|
def _get_str_args(args: list[ast.expr]) -> list[str]:
|
|
str_args = []
|
|
for arg in args:
|
|
assert isinstance(arg, ast.Constant) and isinstance(arg.value, str)
|
|
str_args.append(arg.value)
|
|
return str_args
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class CellMagic:
|
|
name: str
|
|
params: Optional[str]
|
|
body: str
|
|
|
|
@property
|
|
def header(self) -> str:
|
|
if self.params:
|
|
return f"%%{self.name} {self.params}"
|
|
return f"%%{self.name}"
|
|
|
|
|
|
# ast.NodeVisitor + dataclass = breakage under mypyc.
|
|
class CellMagicFinder(ast.NodeVisitor):
|
|
"""Find cell magics.
|
|
|
|
Note that the source of the abstract syntax tree
|
|
will already have been processed by IPython's
|
|
TransformerManager().transform_cell.
|
|
|
|
For example,
|
|
|
|
%%time\n
|
|
foo()
|
|
|
|
would have been transformed to
|
|
|
|
get_ipython().run_cell_magic('time', '', 'foo()\\n')
|
|
|
|
and we look for instances of the latter.
|
|
"""
|
|
|
|
def __init__(self, cell_magic: Optional[CellMagic] = None) -> None:
|
|
self.cell_magic = cell_magic
|
|
|
|
def visit_Expr(self, node: ast.Expr) -> None:
|
|
"""Find cell magic, extract header and body."""
|
|
if (
|
|
isinstance(node.value, ast.Call)
|
|
and _is_ipython_magic(node.value.func)
|
|
and node.value.func.attr == "run_cell_magic"
|
|
):
|
|
args = _get_str_args(node.value.args)
|
|
self.cell_magic = CellMagic(name=args[0], params=args[1], body=args[2])
|
|
self.generic_visit(node)
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class OffsetAndMagic:
|
|
col_offset: int
|
|
magic: str
|
|
|
|
|
|
# Unsurprisingly, subclassing ast.NodeVisitor means we can't use dataclasses here
|
|
# as mypyc will generate broken code.
|
|
class MagicFinder(ast.NodeVisitor):
|
|
"""Visit cell to look for get_ipython calls.
|
|
|
|
Note that the source of the abstract syntax tree
|
|
will already have been processed by IPython's
|
|
TransformerManager().transform_cell.
|
|
|
|
For example,
|
|
|
|
%matplotlib inline
|
|
|
|
would have been transformed to
|
|
|
|
get_ipython().run_line_magic('matplotlib', 'inline')
|
|
|
|
and we look for instances of the latter (and likewise for other
|
|
types of magics).
|
|
"""
|
|
|
|
def __init__(self) -> None:
|
|
self.magics: dict[int, list[OffsetAndMagic]] = collections.defaultdict(list)
|
|
|
|
def visit_Assign(self, node: ast.Assign) -> None:
|
|
"""Look for system assign magics.
|
|
|
|
For example,
|
|
|
|
black_version = !black --version
|
|
env = %env var
|
|
|
|
would have been (respectively) transformed to
|
|
|
|
black_version = get_ipython().getoutput('black --version')
|
|
env = get_ipython().run_line_magic('env', 'var')
|
|
|
|
and we look for instances of any of the latter.
|
|
"""
|
|
if isinstance(node.value, ast.Call) and _is_ipython_magic(node.value.func):
|
|
args = _get_str_args(node.value.args)
|
|
if node.value.func.attr == "getoutput":
|
|
src = f"!{args[0]}"
|
|
elif node.value.func.attr == "run_line_magic":
|
|
src = f"%{args[0]}"
|
|
if args[1]:
|
|
src += f" {args[1]}"
|
|
else:
|
|
raise AssertionError(
|
|
f"Unexpected IPython magic {node.value.func.attr!r} found. "
|
|
"Please report a bug on https://github.com/psf/black/issues."
|
|
) from None
|
|
self.magics[node.value.lineno].append(
|
|
OffsetAndMagic(node.value.col_offset, src)
|
|
)
|
|
self.generic_visit(node)
|
|
|
|
def visit_Expr(self, node: ast.Expr) -> None:
|
|
"""Look for magics in body of cell.
|
|
|
|
For examples,
|
|
|
|
!ls
|
|
!!ls
|
|
?ls
|
|
??ls
|
|
|
|
would (respectively) get transformed to
|
|
|
|
get_ipython().system('ls')
|
|
get_ipython().getoutput('ls')
|
|
get_ipython().run_line_magic('pinfo', 'ls')
|
|
get_ipython().run_line_magic('pinfo2', 'ls')
|
|
|
|
and we look for instances of any of the latter.
|
|
"""
|
|
if isinstance(node.value, ast.Call) and _is_ipython_magic(node.value.func):
|
|
args = _get_str_args(node.value.args)
|
|
if node.value.func.attr == "run_line_magic":
|
|
if args[0] == "pinfo":
|
|
src = f"?{args[1]}"
|
|
elif args[0] == "pinfo2":
|
|
src = f"??{args[1]}"
|
|
else:
|
|
src = f"%{args[0]}"
|
|
if args[1]:
|
|
src += f" {args[1]}"
|
|
elif node.value.func.attr == "system":
|
|
src = f"!{args[0]}"
|
|
elif node.value.func.attr == "getoutput":
|
|
src = f"!!{args[0]}"
|
|
else:
|
|
raise NothingChanged # unsupported magic.
|
|
self.magics[node.value.lineno].append(
|
|
OffsetAndMagic(node.value.col_offset, src)
|
|
)
|
|
self.generic_visit(node)
|