fullgraph=False">

Skipped Functions

Created On: Jul 28, 2025 | Last Updated On: Dec 03, 2025

Summary:

• Sometimes, torch.compile completely gives up compiling a function and runs it eagerly instead, resulting in potentially lost optimization opportunities.

• There are ways to work around skipped functions in order to re-enable tracing around the problematic code.

Sometimes, torch.compile with fullgraph=False is unable to resume tracing when encountering a graph break or other compiler error. In many of these cases, torch.compile will skip compiling the function entirely and run it eagerly.

Note that the skip is only applied to the current function and NOT any nested function calls. torch.compile will still attempt to compile nested calls.

def inner1(x):
    return x + 1
def inner2(x):
    return x + 2
@torch.compile
def fn(x):
    x = inner1(x)
    torch._dynamo.skip_frame()
    x = inner2(x)
fn(torch.randn(3))

ChromiumEventLogger initialized with id a98ec34a-176e-4b8e-a90b-43aec3cb4362
torchdynamo start compiling fn /tmp/ipykernel_503/2126697152.py:5, stack (elided 5 frames):
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
    app.launch_new_instance()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
    app.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
    self.io_loop.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
    self.asyncio_loop.run_forever()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
    self._run_once()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
    handle._run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
    self._context.run(self._callback, *self._args)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
    return await f(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
    await self.dispatch_shell(msg, subshell_id=subshell_id)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
    await result
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
    await super().execute_request(stream, ident, parent)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
    reply_content = await reply_content
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
    res = shell.run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
    return super().run_cell(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
    result = self._run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
    result = runner(coro)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
    coro.send(None)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
    if await self.run_code(code, result, async_=asy):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "/tmp/ipykernel_503/2126697152.py", line 10, in <module>
    fn(torch.randn(3))
Step 1: torchdynamo start tracing fn /tmp/ipykernel_503/2126697152.py:5
create_env
TRACE starts_line /tmp/ipykernel_503/2126697152.py:7 in fn
        x = inner1(x)
TRACE LOAD_GLOBAL inner1 []
TRACE LOAD_FAST x [LazyVariableTracker(unrealized: <class 'function'>)]
TRACE CALL_FUNCTION 1 [LazyVariableTracker(unrealized: <class 'function'>), LazyVariableTracker(unrealized: <class 'torch.Tensor'>)]
INLINING <code object inner1 at 0x7f7fc426f9f0, file "/tmp/ipykernel_503/2126697152.py", line 1>, inlined according trace_rules.lookup inlined by default
wrap_to_fake L['x'] (3,) StatefulSymbolicContext(dynamic_sizes=[<DimDynamic.STATIC: 2>], dynamic_strides=[<DimDynamic.INFER_STRIDE: 4>], constraint_sizes=[None], constraint_strides=[None], specialize_on=[[]], view_base_context=None, tensor_source=LocalSource(local_name='x', is_input=True, dynamism=None, is_derefed_cell_contents=False), shape_env_to_source_to_symbol_cache={}) <class 'torch.Tensor'>
create_graph_input L_x_ L['x'] FakeTensor(..., size=(3,)) at debug_level 0 before=False
TRACE starts_line /tmp/ipykernel_503/2126697152.py:2 in inner1 (inline depth: 1)
        return x + 1
TRACE LOAD_FAST x []
TRACE LOAD_CONST 1 [TensorVariable()]
TRACE BINARY_ADD None [TensorVariable(), ConstantVariable(int: 1)]
TRACE RETURN_VALUE None [TensorVariable()]
DONE INLINING <code object inner1 at 0x7f7fc426f9f0, file "/tmp/ipykernel_503/2126697152.py", line 1>
TRACE STORE_FAST x [TensorVariable()]
TRACE starts_line /tmp/ipykernel_503/2126697152.py:8 in fn
        torch._dynamo.skip_frame()
TRACE LOAD_GLOBAL torch []
TRACE LOAD_ATTR _dynamo [LazyVariableTracker(unrealized: <class 'module'>)]
TRACE LOAD_ATTR skip_frame [LazyVariableTracker(unrealized: <class 'module'>)]
TRACE CALL_FUNCTION 0 [LazyVariableTracker(unrealized: <class 'function'>)]
Graph break in user code at /tmp/ipykernel_503/2126697152.py:8
Graph Break Reason: Failed to handle graph break gracefully. Skipping the function and falling back to eager. Graph break encountered:

Call to `torch._dynamo.skip_frame()`
  Explanation: User-inserted skip frame. Message: 
  Hint: Remove the `torch._dynamo.skip_frame()` call.

  Developer debug context: Called `torch._dynamo.skip_frame()` with args `[]`, kwargs `{}`. Skipping frame fn (/tmp/ipykernel_503/2126697152.py line 5).

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb4297.html

User code traceback:
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
    app.launch_new_instance()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
    app.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
    self.io_loop.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
    self.asyncio_loop.run_forever()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
    self._run_once()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
    handle._run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
    self._context.run(self._callback, *self._args)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
    return await f(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
    await self.dispatch_shell(msg, subshell_id=subshell_id)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
    await result
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
    await super().execute_request(stream, ident, parent)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
    reply_content = await reply_content
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
    res = shell.run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
    return super().run_cell(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
    result = self._run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
    result = runner(coro)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
    coro.send(None)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
    if await self.run_code(code, result, async_=asy):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "/tmp/ipykernel_503/2126697152.py", line 10, in <module>
    fn(torch.randn(3))
  File "/tmp/ipykernel_503/2126697152.py", line 8, in fn
    torch._dynamo.skip_frame()
run_gc_after_compile: running gc
WON'T CONVERT fn /tmp/ipykernel_503/2126697152.py line 5 
due to: 
Traceback (most recent call last):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1975, in __call__
    result = self._inner_convert(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 715, in __call__
    result = _compile(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1772, in _compile
    guarded_code, tracer_output = compile_inner(code, one_graph, hooks)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_utils_internal.py", line 96, in wrapper_function
    return function(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1445, in compile_inner
    return _compile_inner(code, one_graph, hooks)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1479, in _compile_inner
    dynamo_output = compile_frame(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1353, in compile_frame
    bytecode, tracer_output = transform_code_object(code, transform)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/bytecode_transformation.py", line 1608, in transform_code_object
    tracer_output = transformations(instructions, code_options)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1325, in transform
    tracer_output = trace_frame(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 327, in _fn
    return fn(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 849, in trace_frame
    run_tracer()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 830, in run_tracer
    tracer.run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1675, in run
    while self.step():
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1340, in step
    self.dispatch_table[inst.opcode](self, inst)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 888, in wrapper
    return inner_fn(self, inst)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 2605, in CALL_FUNCTION
    self.call_function(fn, args, {})
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1247, in call_function
    self.push(fn.call_function(self, args, kwargs))  # type: ignore[arg-type]
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/variables/lazy.py", line 433, in realize_and_forward
    return getattr(self.realize(), name)(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/variables/functions.py", line 1953, in call_function
    unimplemented(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/exc.py", line 634, in unimplemented
    raise Unsupported(msg, gb_type, skip_frame)
torch._dynamo.exc.Unsupported: Call to `torch._dynamo.skip_frame()`
  Explanation: User-inserted skip frame. Message: 
  Hint: Remove the `torch._dynamo.skip_frame()` call.

  Developer debug context: Called `torch._dynamo.skip_frame()` with args `[]`, kwargs `{}`. Skipping frame fn (/tmp/ipykernel_503/2126697152.py line 5).

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb4297.html

from user code:
   File "/tmp/ipykernel_503/2126697152.py", line 8, in fn
    torch._dynamo.skip_frame()

Set TORCHDYNAMO_VERBOSE=1 for the internal stack trace (please do this especially if you're reporting a bug to PyTorch). For even more developer context, set TORCH_LOGS="+dynamo"

Traceback (most recent call last):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1975, in __call__
    result = self._inner_convert(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 715, in __call__
    result = _compile(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1772, in _compile
    guarded_code, tracer_output = compile_inner(code, one_graph, hooks)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_utils_internal.py", line 96, in wrapper_function
    return function(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1445, in compile_inner
    return _compile_inner(code, one_graph, hooks)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1479, in _compile_inner
    dynamo_output = compile_frame(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1353, in compile_frame
    bytecode, tracer_output = transform_code_object(code, transform)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/bytecode_transformation.py", line 1608, in transform_code_object
    tracer_output = transformations(instructions, code_options)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1325, in transform
    tracer_output = trace_frame(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 327, in _fn
    return fn(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 849, in trace_frame
    run_tracer()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 830, in run_tracer
    tracer.run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1675, in run
    while self.step():
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1340, in step
    self.dispatch_table[inst.opcode](self, inst)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 888, in wrapper
    return inner_fn(self, inst)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 2605, in CALL_FUNCTION
    self.call_function(fn, args, {})
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1247, in call_function
    self.push(fn.call_function(self, args, kwargs))  # type: ignore[arg-type]
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/variables/lazy.py", line 433, in realize_and_forward
    return getattr(self.realize(), name)(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/variables/functions.py", line 1953, in call_function
    unimplemented(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/exc.py", line 634, in unimplemented
    raise Unsupported(msg, gb_type, skip_frame)
torch._dynamo.exc.Unsupported: Call to `torch._dynamo.skip_frame()`
  Explanation: User-inserted skip frame. Message: 
  Hint: Remove the `torch._dynamo.skip_frame()` call.

  Developer debug context: Called `torch._dynamo.skip_frame()` with args `[]`, kwargs `{}`. Skipping frame fn (/tmp/ipykernel_503/2126697152.py line 5).

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb4297.html

from user code:
   File "/tmp/ipykernel_503/2126697152.py", line 8, in fn
    torch._dynamo.skip_frame()

Set TORCHDYNAMO_VERBOSE=1 for the internal stack trace (please do this especially if you're reporting a bug to PyTorch). For even more developer context, set TORCH_LOGS="+dynamo"
torchdynamo start compiling inner1 /tmp/ipykernel_503/2126697152.py:1, stack (elided 5 frames):
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
    app.launch_new_instance()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
    app.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
    self.io_loop.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
    self.asyncio_loop.run_forever()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
    self._run_once()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
    handle._run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
    self._context.run(self._callback, *self._args)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
    return await f(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
    await self.dispatch_shell(msg, subshell_id=subshell_id)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
    await result
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
    await super().execute_request(stream, ident, parent)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
    reply_content = await reply_content
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
    res = shell.run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
    return super().run_cell(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
    result = self._run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
    result = runner(coro)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
    coro.send(None)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
    if await self.run_code(code, result, async_=asy):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "/tmp/ipykernel_503/2126697152.py", line 10, in <module>
    fn(torch.randn(3))
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 992, in compile_wrapper
    return fn(*args, **kwargs)
Step 1: torchdynamo start tracing inner1 /tmp/ipykernel_503/2126697152.py:1
create_env
TRACE starts_line /tmp/ipykernel_503/2126697152.py:2 in inner1
        return x + 1
TRACE LOAD_FAST x []
TRACE LOAD_CONST 1 [LazyVariableTracker(unrealized: <class 'torch.Tensor'>)]
TRACE BINARY_ADD None [LazyVariableTracker(unrealized: <class 'torch.Tensor'>), ConstantVariable(int: 1)]
wrap_to_fake L['x'] (3,) StatefulSymbolicContext(dynamic_sizes=[<DimDynamic.STATIC: 2>], dynamic_strides=[<DimDynamic.INFER_STRIDE: 4>], constraint_sizes=[None], constraint_strides=[None], specialize_on=[[]], view_base_context=None, tensor_source=LocalSource(local_name='x', is_input=True, dynamism=None, is_derefed_cell_contents=False), shape_env_to_source_to_symbol_cache={}) <class 'torch.Tensor'>
create_graph_input L_x_ L['x'] FakeTensor(..., size=(3,)) at debug_level 0 before=False
TRACE RETURN_VALUE None [TensorVariable()]
Step 1: torchdynamo done tracing inner1 (RETURN_VALUE)
return triggered compile
COMPILING GRAPH due to GraphCompileReason(reason='return_value', user_stack=[<FrameSummary file /tmp/ipykernel_503/2126697152.py, line 2 in inner1>], graph_break=False)
TRACED GRAPH
 ===== __compiled_fn_2_7e41af7f_b452_4704_bd8d_eb1ccb917c54 =====
 /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/fx/_lazy_graph_module.py class GraphModule(torch.nn.Module):
    def forward(self, L_x_: "f32[3][1]cpu"):
        l_x_ = L_x_

        # File: /tmp/ipykernel_503/2126697152.py:2 in inner1, code: return x + 1
        add: "f32[3][1]cpu" = l_x_ + 1;  l_x_ = None
        return (add,)
Step 2: calling compiler function eager
Step 2: done compiler function eager
produce_guards
track_symint L['x'].size()[0] 3 None
track_symint L['x'].stride()[0] 1 None
track_symint L['x'].storage_offset() 0 None
Skipping guard L['x'].size()[0] == 3
Skipping guard L['x'].stride()[0] == 1
Skipping guard L['x'].storage_offset() == 0
GUARDS:

TREE_GUARD_MANAGER:
+- RootGuardManager
| +- LAMBDA_GUARD: torch._functorch.aot_autograd.utils.top_saved_tensors_hooks ids == None  # _dynamo/output_graph.py:866 in init_ambient_guards
| +- GLOBAL_STATE: ___check_global_state() against {"allow_bf16_reduce":0,"allow_fp16_reduce":0,"allow_tf32":false,"autocast_state":{"cached_enabled":true,"dtype":[15,5,5,15,5,5,15,15,5,5],"enabled":[false,false,false,false,false,false,false,false,false,false]},"default_dtype":6,"deterministic_algorithms":false,"deterministic_algorithms_warn_only":false,"grad_mode":true,"num_threads":4,"torch_function":true,"torch_function_all_disabled":false}
| +- TORCH_FUNCTION_MODE_STACK: ___check_torch_function_mode_stack()
| +- DEFAULT_DEVICE: utils_device.CURRENT_DEVICE == None                           # _dynamo/output_graph.py:853 in init_ambient_guards
| +- GuardManager: source=L['x'], accessed_by=FrameLocalsGuardAccessor(key='x', framelocals_idx=0), type=<class 'torch.Tensor'>, tag_safe=(True, False)
| | +- TENSOR_MATCH: check_tensor(L['x'], Tensor, DispatchKeySet(CPU, BackendSelect, ADInplaceOrView, AutogradCPU), torch.float32, device=None, requires_grad=False, size=[3], stride=[1])  # return x + 1  # mp/ipykernel_503/2126697152.py:2 in inner1
| | +- NO_HASATTR: hasattr(L['x'], '_dynamo_dynamic_indices') == False           # return x + 1  # mp/ipykernel_503/2126697152.py:2 in inner1
Guard eval latency = 85.07 us
put_code_state: no cache key, skipping
run_gc_after_compile: running gc
skipping: inner (reason: in skipfiles, file: /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_compile.py)
skipping: disable (reason: in skipfiles, file: /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/decorators.py)
skipping: innermost_fn (reason: in skipfiles, file: /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py)
skipping: __init__ (reason: in skipfiles, file: /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py)
skipping: __init__ (reason: in skipfiles, file: /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py)
skipping: nothing (reason: in skipfiles, file: /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py)
skipping: __call__ (reason: in skipfiles, file: /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py)
skipping: _fn (reason: in skipfiles, file: /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py)
skipping: skip_frame (reason: in skipfiles, file: /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/decorators.py)
torchdynamo start compiling inner2 /tmp/ipykernel_503/2126697152.py:3, stack (elided 5 frames):
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
    app.launch_new_instance()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
    app.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
    self.io_loop.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
    self.asyncio_loop.run_forever()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
    self._run_once()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
    handle._run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
    self._context.run(self._callback, *self._args)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
    return await f(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
    await self.dispatch_shell(msg, subshell_id=subshell_id)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
    await result
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
    await super().execute_request(stream, ident, parent)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
    reply_content = await reply_content
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
    res = shell.run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
    return super().run_cell(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
    result = self._run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
    result = runner(coro)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
    coro.send(None)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
    if await self.run_code(code, result, async_=asy):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "/tmp/ipykernel_503/2126697152.py", line 10, in <module>
    fn(torch.randn(3))
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 992, in compile_wrapper
    return fn(*args, **kwargs)
Step 1: torchdynamo start tracing inner2 /tmp/ipykernel_503/2126697152.py:3
create_env
TRACE starts_line /tmp/ipykernel_503/2126697152.py:4 in inner2
        return x + 2
TRACE LOAD_FAST x []
TRACE LOAD_CONST 2 [LazyVariableTracker(unrealized: <class 'torch.Tensor'>)]
TRACE BINARY_ADD None [LazyVariableTracker(unrealized: <class 'torch.Tensor'>), ConstantVariable(int: 2)]
wrap_to_fake L['x'] (3,) StatefulSymbolicContext(dynamic_sizes=[<DimDynamic.STATIC: 2>], dynamic_strides=[<DimDynamic.INFER_STRIDE: 4>], constraint_sizes=[None], constraint_strides=[None], specialize_on=[[]], view_base_context=None, tensor_source=LocalSource(local_name='x', is_input=True, dynamism=None, is_derefed_cell_contents=False), shape_env_to_source_to_symbol_cache={}) <class 'torch.Tensor'>
create_graph_input L_x_ L['x'] FakeTensor(..., size=(3,)) at debug_level 0 before=False
TRACE RETURN_VALUE None [TensorVariable()]
Step 1: torchdynamo done tracing inner2 (RETURN_VALUE)
return triggered compile
COMPILING GRAPH due to GraphCompileReason(reason='return_value', user_stack=[<FrameSummary file /tmp/ipykernel_503/2126697152.py, line 4 in inner2>], graph_break=False)
TRACED GRAPH
 ===== __compiled_fn_4_a2d9a276_5acf_458f_992e_d8a68c85a6a9 =====
 /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/fx/_lazy_graph_module.py class GraphModule(torch.nn.Module):
    def forward(self, L_x_: "f32[3][1]cpu"):
        l_x_ = L_x_

        # File: /tmp/ipykernel_503/2126697152.py:4 in inner2, code: return x + 2
        add: "f32[3][1]cpu" = l_x_ + 2;  l_x_ = None
        return (add,)
Step 2: calling compiler function eager
Step 2: done compiler function eager
produce_guards
track_symint L['x'].size()[0] 3 None
track_symint L['x'].stride()[0] 1 None
track_symint L['x'].storage_offset() 0 None
Skipping guard L['x'].size()[0] == 3
Skipping guard L['x'].stride()[0] == 1
Skipping guard L['x'].storage_offset() == 0
GUARDS:

TREE_GUARD_MANAGER:
+- RootGuardManager
| +- LAMBDA_GUARD: torch._functorch.aot_autograd.utils.top_saved_tensors_hooks ids == None  # _dynamo/output_graph.py:866 in init_ambient_guards
| +- GLOBAL_STATE: ___check_global_state() against {"allow_bf16_reduce":0,"allow_fp16_reduce":0,"allow_tf32":false,"autocast_state":{"cached_enabled":true,"dtype":[15,5,5,15,5,5,15,15,5,5],"enabled":[false,false,false,false,false,false,false,false,false,false]},"default_dtype":6,"deterministic_algorithms":false,"deterministic_algorithms_warn_only":false,"grad_mode":true,"num_threads":4,"torch_function":true,"torch_function_all_disabled":false}
| +- TORCH_FUNCTION_MODE_STACK: ___check_torch_function_mode_stack()
| +- DEFAULT_DEVICE: utils_device.CURRENT_DEVICE == None                           # _dynamo/output_graph.py:853 in init_ambient_guards
| +- GuardManager: source=L['x'], accessed_by=FrameLocalsGuardAccessor(key='x', framelocals_idx=0), type=<class 'torch.Tensor'>, tag_safe=(True, False)
| | +- TENSOR_MATCH: check_tensor(L['x'], Tensor, DispatchKeySet(CPU, BackendSelect, ADInplaceOrView, AutogradCPU), torch.float32, device=None, requires_grad=False, size=[3], stride=[1])  # return x + 2  # mp/ipykernel_503/2126697152.py:4 in inner2
| | +- NO_HASATTR: hasattr(L['x'], '_dynamo_dynamic_indices') == False           # return x + 2  # mp/ipykernel_503/2126697152.py:4 in inner2
Guard eval latency = 38.28 us
put_code_state: no cache key, skipping
run_gc_after_compile: running gc
skipping: remove (reason: in skipfiles, file: /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/utils/weak.py)
skipping: __hash__ (reason: in skipfiles, file: /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/utils/weak.py)

In the above example, torch.compile will trace fn (including inner1) up until the skip_frame. Then fn is skipped and run eagerly - inner1 and inner2 are compiled when they are called.

Skipping functions may result in lost optimization opportunities, so it is important to check if code you want compiled is being skipped, and if so, to work around the skip.

Graph Break in a Loop

torch.compile cannot resume tracing if a graph break occurs in a loop:

@torch.compile
def fn(x):
    for i in range(5):
        x = x + 1
        if i == 3:
            torch._dynamo.graph_break()
    return x
fn(torch.randn(3))

torchdynamo start compiling fn /tmp/ipykernel_503/2044822433.py:1, stack (elided 5 frames):
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
    app.launch_new_instance()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
    app.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
    self.io_loop.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
    self.asyncio_loop.run_forever()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
    self._run_once()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
    handle._run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
    self._context.run(self._callback, *self._args)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
    return await f(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
    await self.dispatch_shell(msg, subshell_id=subshell_id)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
    await result
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
    await super().execute_request(stream, ident, parent)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
    reply_content = await reply_content
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
    res = shell.run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
    return super().run_cell(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
    result = self._run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
    result = runner(coro)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
    coro.send(None)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
    if await self.run_code(code, result, async_=asy):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "/tmp/ipykernel_503/2044822433.py", line 8, in <module>
    fn(torch.randn(3))
Step 1: torchdynamo start tracing fn /tmp/ipykernel_503/2044822433.py:1
create_env
TRACE starts_line /tmp/ipykernel_503/2044822433.py:3 in fn
        for i in range(5):
TRACE LOAD_GLOBAL range []
TRACE LOAD_CONST 5 [LazyVariableTracker(unrealized: <class 'type'>)]
TRACE CALL_FUNCTION 1 [LazyVariableTracker(unrealized: <class 'type'>), ConstantVariable(int: 5)]
TRACE GET_ITER None [RangeVariable()]
TRACE FOR_ITER 40 [RangeIteratorVariable()]
TRACE STORE_FAST i [RangeIteratorVariable(), ConstantVariable(int: 0)]
TRACE starts_line /tmp/ipykernel_503/2044822433.py:4 in fn
            x = x + 1
TRACE LOAD_FAST x [RangeIteratorVariable()]
TRACE LOAD_CONST 1 [RangeIteratorVariable(), LazyVariableTracker(unrealized: <class 'torch.Tensor'>)]
TRACE BINARY_ADD None [RangeIteratorVariable(), LazyVariableTracker(unrealized: <class 'torch.Tensor'>), ConstantVariable(int: 1)]
wrap_to_fake L['x'] (3,) StatefulSymbolicContext(dynamic_sizes=[<DimDynamic.STATIC: 2>], dynamic_strides=[<DimDynamic.INFER_STRIDE: 4>], constraint_sizes=[None], constraint_strides=[None], specialize_on=[[]], view_base_context=None, tensor_source=LocalSource(local_name='x', is_input=True, dynamism=None, is_derefed_cell_contents=False), shape_env_to_source_to_symbol_cache={}) <class 'torch.Tensor'>
create_graph_input L_x_ L['x'] FakeTensor(..., size=(3,)) at debug_level 0 before=False
TRACE STORE_FAST x [RangeIteratorVariable(), TensorVariable()]
TRACE starts_line /tmp/ipykernel_503/2044822433.py:5 in fn
            if i == 3:
TRACE LOAD_FAST i [RangeIteratorVariable()]
TRACE LOAD_CONST 3 [RangeIteratorVariable(), ConstantVariable(int: 0)]
TRACE COMPARE_OP == [RangeIteratorVariable(), ConstantVariable(int: 0), ConstantVariable(int: 3)]
TRACE POP_JUMP_IF_FALSE 38 [RangeIteratorVariable(), ConstantVariable(bool: False)]
TRACE starts_line /tmp/ipykernel_503/2044822433.py:6 in fn
                torch._dynamo.graph_break()
TRACE JUMP_ABSOLUTE 8 [RangeIteratorVariable()]
TRACE starts_line /tmp/ipykernel_503/2044822433.py:3 in fn
        for i in range(5):
TRACE FOR_ITER 40 [RangeIteratorVariable()]
TRACE STORE_FAST i [RangeIteratorVariable(), ConstantVariable(int: 1)]
TRACE starts_line /tmp/ipykernel_503/2044822433.py:4 in fn
            x = x + 1
TRACE LOAD_FAST x [RangeIteratorVariable()]
TRACE LOAD_CONST 1 [RangeIteratorVariable(), TensorVariable()]
TRACE BINARY_ADD None [RangeIteratorVariable(), TensorVariable(), ConstantVariable(int: 1)]
TRACE STORE_FAST x [RangeIteratorVariable(), TensorVariable()]
TRACE starts_line /tmp/ipykernel_503/2044822433.py:5 in fn
            if i == 3:
TRACE LOAD_FAST i [RangeIteratorVariable()]
TRACE LOAD_CONST 3 [RangeIteratorVariable(), ConstantVariable(int: 1)]
TRACE COMPARE_OP == [RangeIteratorVariable(), ConstantVariable(int: 1), ConstantVariable(int: 3)]
TRACE POP_JUMP_IF_FALSE 38 [RangeIteratorVariable(), ConstantVariable(bool: False)]
TRACE starts_line /tmp/ipykernel_503/2044822433.py:6 in fn
                torch._dynamo.graph_break()
TRACE JUMP_ABSOLUTE 8 [RangeIteratorVariable()]
TRACE starts_line /tmp/ipykernel_503/2044822433.py:3 in fn
        for i in range(5):
TRACE FOR_ITER 40 [RangeIteratorVariable()]
TRACE STORE_FAST i [RangeIteratorVariable(), ConstantVariable(int: 2)]
TRACE starts_line /tmp/ipykernel_503/2044822433.py:4 in fn
            x = x + 1
TRACE LOAD_FAST x [RangeIteratorVariable()]
TRACE LOAD_CONST 1 [RangeIteratorVariable(), TensorVariable()]
TRACE BINARY_ADD None [RangeIteratorVariable(), TensorVariable(), ConstantVariable(int: 1)]
TRACE STORE_FAST x [RangeIteratorVariable(), TensorVariable()]
TRACE starts_line /tmp/ipykernel_503/2044822433.py:5 in fn
            if i == 3:
TRACE LOAD_FAST i [RangeIteratorVariable()]
TRACE LOAD_CONST 3 [RangeIteratorVariable(), ConstantVariable(int: 2)]
TRACE COMPARE_OP == [RangeIteratorVariable(), ConstantVariable(int: 2), ConstantVariable(int: 3)]
TRACE POP_JUMP_IF_FALSE 38 [RangeIteratorVariable(), ConstantVariable(bool: False)]
TRACE starts_line /tmp/ipykernel_503/2044822433.py:6 in fn
                torch._dynamo.graph_break()
TRACE JUMP_ABSOLUTE 8 [RangeIteratorVariable()]
TRACE starts_line /tmp/ipykernel_503/2044822433.py:3 in fn
        for i in range(5):
TRACE FOR_ITER 40 [RangeIteratorVariable()]
TRACE STORE_FAST i [RangeIteratorVariable(), ConstantVariable(int: 3)]
TRACE starts_line /tmp/ipykernel_503/2044822433.py:4 in fn
            x = x + 1
TRACE LOAD_FAST x [RangeIteratorVariable()]
TRACE LOAD_CONST 1 [RangeIteratorVariable(), TensorVariable()]
TRACE BINARY_ADD None [RangeIteratorVariable(), TensorVariable(), ConstantVariable(int: 1)]
TRACE STORE_FAST x [RangeIteratorVariable(), TensorVariable()]
TRACE starts_line /tmp/ipykernel_503/2044822433.py:5 in fn
            if i == 3:
TRACE LOAD_FAST i [RangeIteratorVariable()]
TRACE LOAD_CONST 3 [RangeIteratorVariable(), ConstantVariable(int: 3)]
TRACE COMPARE_OP == [RangeIteratorVariable(), ConstantVariable(int: 3), ConstantVariable(int: 3)]
TRACE POP_JUMP_IF_FALSE 38 [RangeIteratorVariable(), ConstantVariable(bool: True)]
TRACE starts_line /tmp/ipykernel_503/2044822433.py:6 in fn
                torch._dynamo.graph_break()
TRACE LOAD_GLOBAL torch [RangeIteratorVariable()]
TRACE LOAD_ATTR _dynamo [RangeIteratorVariable(), LazyVariableTracker(unrealized: <class 'module'>)]
TRACE LOAD_ATTR graph_break [RangeIteratorVariable(), LazyVariableTracker(unrealized: <class 'module'>)]
TRACE CALL_FUNCTION 0 [RangeIteratorVariable(), LazyVariableTracker(unrealized: <class 'function'>)]
Graph break in user code at /tmp/ipykernel_503/2044822433.py:6
Graph Break Reason: Failed to handle graph break gracefully. Skipping the function and falling back to eager. Graph break encountered:

Call to `torch._dynamo.graph_break()`
  Explanation: User-inserted graph break. Message: None
  Hint: Remove the `torch._dynamo.graph_break()` call.

  Developer debug context: Called `torch._dynamo.graph_break()` with args `[]`, kwargs `{}`

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb0025.html

*** While handling this graph break, another graph break occurred: ***

graph break in loop
  Explanation: torch.compile detected a graph break in a for/while loop. Skipping the frame and falling back to eager, as graph breaks in loops are not supported.
  Hint: This graph break may have been caused by an earlier graph break. Resolving the earlier graph break may resolve this one.

  Developer debug context: frame skipped: fn (/tmp/ipykernel_503/2044822433.py line 1)

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb7000.html

User code traceback:
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
    app.launch_new_instance()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
    app.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
    self.io_loop.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
    self.asyncio_loop.run_forever()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
    self._run_once()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
    handle._run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
    self._context.run(self._callback, *self._args)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
    return await f(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
    await self.dispatch_shell(msg, subshell_id=subshell_id)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
    await result
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
    await super().execute_request(stream, ident, parent)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
    reply_content = await reply_content
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
    res = shell.run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
    return super().run_cell(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
    result = self._run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
    result = runner(coro)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
    coro.send(None)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
    if await self.run_code(code, result, async_=asy):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "/tmp/ipykernel_503/2044822433.py", line 8, in <module>
    fn(torch.randn(3))
  File "/tmp/ipykernel_503/2044822433.py", line 6, in fn
    torch._dynamo.graph_break()
run_gc_after_compile: running gc
WON'T CONVERT fn /tmp/ipykernel_503/2044822433.py line 1 
due to: 
Traceback (most recent call last):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 888, in wrapper
    return inner_fn(self, inst)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 2605, in CALL_FUNCTION
    self.call_function(fn, args, {})
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1247, in call_function
    self.push(fn.call_function(self, args, kwargs))  # type: ignore[arg-type]
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/variables/lazy.py", line 433, in realize_and_forward
    return getattr(self.realize(), name)(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/variables/functions.py", line 1939, in call_function
    unimplemented(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/exc.py", line 634, in unimplemented
    raise Unsupported(msg, gb_type, skip_frame)
torch._dynamo.exc.Unsupported: Call to `torch._dynamo.graph_break()`
  Explanation: User-inserted graph break. Message: None
  Hint: Remove the `torch._dynamo.graph_break()` call.

  Developer debug context: Called `torch._dynamo.graph_break()` with args `[]`, kwargs `{}`

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb0025.html

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1975, in __call__
    result = self._inner_convert(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 715, in __call__
    result = _compile(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1772, in _compile
    guarded_code, tracer_output = compile_inner(code, one_graph, hooks)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_utils_internal.py", line 96, in wrapper_function
    return function(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1445, in compile_inner
    return _compile_inner(code, one_graph, hooks)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1479, in _compile_inner
    dynamo_output = compile_frame(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1353, in compile_frame
    bytecode, tracer_output = transform_code_object(code, transform)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/bytecode_transformation.py", line 1608, in transform_code_object
    tracer_output = transformations(instructions, code_options)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1325, in transform
    tracer_output = trace_frame(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 327, in _fn
    return fn(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 849, in trace_frame
    run_tracer()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 830, in run_tracer
    tracer.run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1675, in run
    while self.step():
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1340, in step
    self.dispatch_table[inst.opcode](self, inst)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 916, in wrapper
    self.raise_loop_graph_break(self.f_code, excp)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 4398, in raise_loop_graph_break
    unimplemented(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/exc.py", line 630, in unimplemented
    raise Unsupported(msg, gb_type, skip_frame, real_stack=past_real_stack)
torch._dynamo.exc.Unsupported: Call to `torch._dynamo.graph_break()`
  Explanation: User-inserted graph break. Message: None
  Hint: Remove the `torch._dynamo.graph_break()` call.

  Developer debug context: Called `torch._dynamo.graph_break()` with args `[]`, kwargs `{}`

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb0025.html

*** While handling this graph break, another graph break occurred: ***

graph break in loop
  Explanation: torch.compile detected a graph break in a for/while loop. Skipping the frame and falling back to eager, as graph breaks in loops are not supported.
  Hint: This graph break may have been caused by an earlier graph break. Resolving the earlier graph break may resolve this one.

  Developer debug context: frame skipped: fn (/tmp/ipykernel_503/2044822433.py line 1)

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb7000.html

from user code:
   File "/tmp/ipykernel_503/2044822433.py", line 6, in fn
    torch._dynamo.graph_break()

Set TORCHDYNAMO_VERBOSE=1 for the internal stack trace (please do this especially if you're reporting a bug to PyTorch). For even more developer context, set TORCH_LOGS="+dynamo"

Traceback (most recent call last):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 888, in wrapper
    return inner_fn(self, inst)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 2605, in CALL_FUNCTION
    self.call_function(fn, args, {})
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1247, in call_function
    self.push(fn.call_function(self, args, kwargs))  # type: ignore[arg-type]
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/variables/lazy.py", line 433, in realize_and_forward
    return getattr(self.realize(), name)(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/variables/functions.py", line 1939, in call_function
    unimplemented(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/exc.py", line 634, in unimplemented
    raise Unsupported(msg, gb_type, skip_frame)
torch._dynamo.exc.Unsupported: Call to `torch._dynamo.graph_break()`
  Explanation: User-inserted graph break. Message: None
  Hint: Remove the `torch._dynamo.graph_break()` call.

  Developer debug context: Called `torch._dynamo.graph_break()` with args `[]`, kwargs `{}`

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb0025.html

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1975, in __call__
    result = self._inner_convert(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 715, in __call__
    result = _compile(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1772, in _compile
    guarded_code, tracer_output = compile_inner(code, one_graph, hooks)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_utils_internal.py", line 96, in wrapper_function
    return function(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1445, in compile_inner
    return _compile_inner(code, one_graph, hooks)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1479, in _compile_inner
    dynamo_output = compile_frame(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1353, in compile_frame
    bytecode, tracer_output = transform_code_object(code, transform)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/bytecode_transformation.py", line 1608, in transform_code_object
    tracer_output = transformations(instructions, code_options)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1325, in transform
    tracer_output = trace_frame(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 327, in _fn
    return fn(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 849, in trace_frame
    run_tracer()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 830, in run_tracer
    tracer.run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1675, in run
    while self.step():
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1340, in step
    self.dispatch_table[inst.opcode](self, inst)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 916, in wrapper
    self.raise_loop_graph_break(self.f_code, excp)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 4398, in raise_loop_graph_break
    unimplemented(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/exc.py", line 630, in unimplemented
    raise Unsupported(msg, gb_type, skip_frame, real_stack=past_real_stack)
torch._dynamo.exc.Unsupported: Call to `torch._dynamo.graph_break()`
  Explanation: User-inserted graph break. Message: None
  Hint: Remove the `torch._dynamo.graph_break()` call.

  Developer debug context: Called `torch._dynamo.graph_break()` with args `[]`, kwargs `{}`

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb0025.html

*** While handling this graph break, another graph break occurred: ***

graph break in loop
  Explanation: torch.compile detected a graph break in a for/while loop. Skipping the frame and falling back to eager, as graph breaks in loops are not supported.
  Hint: This graph break may have been caused by an earlier graph break. Resolving the earlier graph break may resolve this one.

  Developer debug context: frame skipped: fn (/tmp/ipykernel_503/2044822433.py line 1)

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb7000.html

from user code:
   File "/tmp/ipykernel_503/2044822433.py", line 6, in fn
    torch._dynamo.graph_break()

Set TORCHDYNAMO_VERBOSE=1 for the internal stack trace (please do this especially if you're reporting a bug to PyTorch). For even more developer context, set TORCH_LOGS="+dynamo"
skipping: graph_break (reason: in skipfiles, file: /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/decorators.py)

tensor([6.6624, 6.1275, 6.4685])

In this example, we can avoid skipping by unrolling the loop:

@torch.compile
def fn(x):
    def inner(i):
        nonlocal x
        x = x + 1
        if i == 3:
            torch._dynamo.graph_break()
    inner(0)
    inner(1)
    inner(2)
    inner(3)
    inner(4)
    return x
fn(torch.randn(3))

torchdynamo start compiling fn /tmp/ipykernel_503/617960493.py:1, stack (elided 5 frames):
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
    app.launch_new_instance()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
    app.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
    self.io_loop.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
    self.asyncio_loop.run_forever()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
    self._run_once()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
    handle._run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
    self._context.run(self._callback, *self._args)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
    return await f(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
    await self.dispatch_shell(msg, subshell_id=subshell_id)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
    await result
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
    await super().execute_request(stream, ident, parent)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
    reply_content = await reply_content
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
    res = shell.run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
    return super().run_cell(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
    result = self._run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
    result = runner(coro)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
    coro.send(None)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
    if await self.run_code(code, result, async_=asy):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "/tmp/ipykernel_503/617960493.py", line 14, in <module>
    fn(torch.randn(3))
Step 1: torchdynamo start tracing fn /tmp/ipykernel_503/617960493.py:1
create_env
TRACE starts_line /tmp/ipykernel_503/617960493.py:3 in fn
        def inner(i):
TRACE LOAD_CLOSURE x []
TRACE BUILD_TUPLE 1 [CellVariable()]
TRACE LOAD_CONST <code object inner at 0x7f7f5ddf8f50, file "/tmp/ipykernel_503/617960493.py", line 3> [TupleVariable(length=1)]
TRACE LOAD_CONST fn.<locals>.inner [TupleVariable(length=1), ConstantVariable(code: <code object inner at 0x7f7f5ddf8f50, file "/tmp/ipykernel_503/617960493.py", line 3>)]
TRACE MAKE_FUNCTION 8 [TupleVariable(length=1), ConstantVariable(code: <code object inner at 0x7f7f5ddf8f50, file "/tmp/ipykernel_503/617960493.py", line 3>), ConstantVariable(str: 'fn.<locals>.inner')]
TRACE STORE_FAST inner [NestedUserFunctionVariable()]
TRACE starts_line /tmp/ipykernel_503/617960493.py:8 in fn
        inner(0)
TRACE LOAD_FAST inner []
TRACE LOAD_CONST 0 [NestedUserFunctionVariable()]
TRACE CALL_FUNCTION 1 [NestedUserFunctionVariable(), ConstantVariable(int: 0)]
wrap_to_fake L['x'] (3,) StatefulSymbolicContext(dynamic_sizes=[<DimDynamic.STATIC: 2>], dynamic_strides=[<DimDynamic.INFER_STRIDE: 4>], constraint_sizes=[None], constraint_strides=[None], specialize_on=[[]], view_base_context=None, tensor_source=LocalSource(local_name='x', is_input=True, dynamism=None, is_derefed_cell_contents=True), shape_env_to_source_to_symbol_cache={}) <class 'torch.Tensor'>
create_graph_input L_x_ L['x'] FakeTensor(..., size=(3,)) at debug_level 0 before=False
INLINING <code object inner at 0x7f7f5ddf8f50, file "/tmp/ipykernel_503/617960493.py", line 3>, inlined according trace_rules.lookup inlined by default
TRACE starts_line /tmp/ipykernel_503/617960493.py:5 in inner (inline depth: 1)
            x = x + 1
TRACE LOAD_DEREF x []
TRACE LOAD_CONST 1 [LazyVariableTracker(realized: TensorVariable())]
TRACE BINARY_ADD None [LazyVariableTracker(realized: TensorVariable()), ConstantVariable(int: 1)]
TRACE STORE_DEREF x [TensorVariable()]
TRACE starts_line /tmp/ipykernel_503/617960493.py:6 in inner (inline depth: 1)
            if i == 3:
TRACE LOAD_FAST i []
TRACE LOAD_CONST 3 [ConstantVariable(int: 0)]
TRACE COMPARE_OP == [ConstantVariable(int: 0), ConstantVariable(int: 3)]
TRACE POP_JUMP_IF_FALSE 30 [ConstantVariable(bool: False)]
TRACE LOAD_CONST None []
TRACE RETURN_VALUE None [ConstantVariable(NoneType: None)]
DONE INLINING <code object inner at 0x7f7f5ddf8f50, file "/tmp/ipykernel_503/617960493.py", line 3>
TRACE POP_TOP None [ConstantVariable(NoneType: None)]
TRACE starts_line /tmp/ipykernel_503/617960493.py:9 in fn
        inner(1)
TRACE LOAD_FAST inner []
TRACE LOAD_CONST 1 [NestedUserFunctionVariable()]
TRACE CALL_FUNCTION 1 [NestedUserFunctionVariable(), ConstantVariable(int: 1)]
INLINING <code object inner at 0x7f7f5ddf8f50, file "/tmp/ipykernel_503/617960493.py", line 3>, inlined according trace_rules.lookup inlined by default
TRACE starts_line /tmp/ipykernel_503/617960493.py:5 in inner (inline depth: 1)
            x = x + 1
TRACE LOAD_DEREF x []
TRACE LOAD_CONST 1 [TensorVariable()]
TRACE BINARY_ADD None [TensorVariable(), ConstantVariable(int: 1)]
TRACE STORE_DEREF x [TensorVariable()]
TRACE starts_line /tmp/ipykernel_503/617960493.py:6 in inner (inline depth: 1)
            if i == 3:
TRACE LOAD_FAST i []
TRACE LOAD_CONST 3 [ConstantVariable(int: 1)]
TRACE COMPARE_OP == [ConstantVariable(int: 1), ConstantVariable(int: 3)]
TRACE POP_JUMP_IF_FALSE 30 [ConstantVariable(bool: False)]
TRACE LOAD_CONST None []
TRACE RETURN_VALUE None [ConstantVariable(NoneType: None)]
DONE INLINING <code object inner at 0x7f7f5ddf8f50, file "/tmp/ipykernel_503/617960493.py", line 3>
TRACE POP_TOP None [ConstantVariable(NoneType: None)]
TRACE starts_line /tmp/ipykernel_503/617960493.py:10 in fn
        inner(2)
TRACE LOAD_FAST inner []
TRACE LOAD_CONST 2 [NestedUserFunctionVariable()]
TRACE CALL_FUNCTION 1 [NestedUserFunctionVariable(), ConstantVariable(int: 2)]
INLINING <code object inner at 0x7f7f5ddf8f50, file "/tmp/ipykernel_503/617960493.py", line 3>, inlined according trace_rules.lookup inlined by default
TRACE starts_line /tmp/ipykernel_503/617960493.py:5 in inner (inline depth: 1)
            x = x + 1
TRACE LOAD_DEREF x []
TRACE LOAD_CONST 1 [TensorVariable()]
TRACE BINARY_ADD None [TensorVariable(), ConstantVariable(int: 1)]
TRACE STORE_DEREF x [TensorVariable()]
TRACE starts_line /tmp/ipykernel_503/617960493.py:6 in inner (inline depth: 1)
            if i == 3:
TRACE LOAD_FAST i []
TRACE LOAD_CONST 3 [ConstantVariable(int: 2)]
TRACE COMPARE_OP == [ConstantVariable(int: 2), ConstantVariable(int: 3)]
TRACE POP_JUMP_IF_FALSE 30 [ConstantVariable(bool: False)]
TRACE LOAD_CONST None []
TRACE RETURN_VALUE None [ConstantVariable(NoneType: None)]
DONE INLINING <code object inner at 0x7f7f5ddf8f50, file "/tmp/ipykernel_503/617960493.py", line 3>
TRACE POP_TOP None [ConstantVariable(NoneType: None)]
TRACE starts_line /tmp/ipykernel_503/617960493.py:11 in fn
        inner(3)
TRACE LOAD_FAST inner []
TRACE LOAD_CONST 3 [NestedUserFunctionVariable()]
TRACE CALL_FUNCTION 1 [NestedUserFunctionVariable(), ConstantVariable(int: 3)]
INLINING <code object inner at 0x7f7f5ddf8f50, file "/tmp/ipykernel_503/617960493.py", line 3>, inlined according trace_rules.lookup inlined by default
TRACE starts_line /tmp/ipykernel_503/617960493.py:5 in inner (inline depth: 1)
            x = x + 1
TRACE LOAD_DEREF x []
TRACE LOAD_CONST 1 [TensorVariable()]
TRACE BINARY_ADD None [TensorVariable(), ConstantVariable(int: 1)]
TRACE STORE_DEREF x [TensorVariable()]
TRACE starts_line /tmp/ipykernel_503/617960493.py:6 in inner (inline depth: 1)
            if i == 3:
TRACE LOAD_FAST i []
TRACE LOAD_CONST 3 [ConstantVariable(int: 3)]
TRACE COMPARE_OP == [ConstantVariable(int: 3), ConstantVariable(int: 3)]
TRACE POP_JUMP_IF_FALSE 30 [ConstantVariable(bool: True)]
TRACE starts_line /tmp/ipykernel_503/617960493.py:7 in inner (inline depth: 1)
                torch._dynamo.graph_break()
TRACE LOAD_GLOBAL torch []
TRACE LOAD_ATTR _dynamo [LazyVariableTracker(unrealized: <class 'module'>)]
TRACE LOAD_ATTR graph_break [LazyVariableTracker(unrealized: <class 'module'>)]
TRACE CALL_FUNCTION 0 [LazyVariableTracker(unrealized: <class 'function'>)]
empty checkpoint - cannot resume from graph break
Graph break in user code at /tmp/ipykernel_503/617960493.py:7
Graph Break Reason: Encountered graph break when attempting to trace CALL_FUNCTION: a call to a regular function, e.g. f(x, y):

Call to `torch._dynamo.graph_break()`
  Explanation: User-inserted graph break. Message: None
  Hint: Remove the `torch._dynamo.graph_break()` call.

  Developer debug context: Called `torch._dynamo.graph_break()` with args `[]`, kwargs `{}`

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb0025.html

User code traceback:
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
    app.launch_new_instance()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
    app.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
    self.io_loop.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
    self.asyncio_loop.run_forever()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
    self._run_once()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
    handle._run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
    self._context.run(self._callback, *self._args)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
    return await f(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
    await self.dispatch_shell(msg, subshell_id=subshell_id)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
    await result
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
    await super().execute_request(stream, ident, parent)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
    reply_content = await reply_content
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
    res = shell.run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
    return super().run_cell(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
    result = self._run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
    result = runner(coro)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
    coro.send(None)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
    if await self.run_code(code, result, async_=asy):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "/tmp/ipykernel_503/617960493.py", line 14, in <module>
    fn(torch.randn(3))
  File "/tmp/ipykernel_503/617960493.py", line 11, in fn
    inner(3)
  File "/tmp/ipykernel_503/617960493.py", line 7, in inner
    torch._dynamo.graph_break()
Restarting analysis due to _dynamo/symbolic_convert.py:252 in fail_and_restart_analysis
Step 1: torchdynamo start tracing fn /tmp/ipykernel_503/617960493.py:1
create_env
TRACE starts_line /tmp/ipykernel_503/617960493.py:3 in fn
        def inner(i):
TRACE LOAD_CLOSURE x []
TRACE BUILD_TUPLE 1 [CellVariable()]
TRACE LOAD_CONST <code object inner at 0x7f7f5ddf8f50, file "/tmp/ipykernel_503/617960493.py", line 3> [TupleVariable(length=1)]
TRACE LOAD_CONST fn.<locals>.inner [TupleVariable(length=1), ConstantVariable(code: <code object inner at 0x7f7f5ddf8f50, file "/tmp/ipykernel_503/617960493.py", line 3>)]
TRACE MAKE_FUNCTION 8 [TupleVariable(length=1), ConstantVariable(code: <code object inner at 0x7f7f5ddf8f50, file "/tmp/ipykernel_503/617960493.py", line 3>), ConstantVariable(str: 'fn.<locals>.inner')]
TRACE STORE_FAST inner [NestedUserFunctionVariable()]
TRACE starts_line /tmp/ipykernel_503/617960493.py:8 in fn
        inner(0)
TRACE LOAD_FAST inner []
TRACE LOAD_CONST 0 [NestedUserFunctionVariable()]
TRACE CALL_FUNCTION 1 [NestedUserFunctionVariable(), ConstantVariable(int: 0)]
wrap_to_fake L['x'] (3,) StatefulSymbolicContext(dynamic_sizes=[<DimDynamic.STATIC: 2>], dynamic_strides=[<DimDynamic.INFER_STRIDE: 4>], constraint_sizes=[None], constraint_strides=[None], specialize_on=[[]], view_base_context=None, tensor_source=LocalSource(local_name='x', is_input=True, dynamism=None, is_derefed_cell_contents=True), shape_env_to_source_to_symbol_cache={}) <class 'torch.Tensor'>
create_graph_input L_x_ L['x'] FakeTensor(..., size=(3,)) at debug_level 0 before=False
INLINING <code object inner at 0x7f7f5ddf8f50, file "/tmp/ipykernel_503/617960493.py", line 3>, inlined according trace_rules.lookup inlined by default
TRACE starts_line /tmp/ipykernel_503/617960493.py:5 in inner (inline depth: 1)
            x = x + 1
TRACE LOAD_DEREF x []
TRACE LOAD_CONST 1 [LazyVariableTracker(realized: TensorVariable())]
TRACE BINARY_ADD None [LazyVariableTracker(realized: TensorVariable()), ConstantVariable(int: 1)]
TRACE STORE_DEREF x [TensorVariable()]
TRACE starts_line /tmp/ipykernel_503/617960493.py:6 in inner (inline depth: 1)
            if i == 3:
TRACE LOAD_FAST i []
TRACE LOAD_CONST 3 [ConstantVariable(int: 0)]
TRACE COMPARE_OP == [ConstantVariable(int: 0), ConstantVariable(int: 3)]
TRACE POP_JUMP_IF_FALSE 30 [ConstantVariable(bool: False)]
TRACE LOAD_CONST None []
TRACE RETURN_VALUE None [ConstantVariable(NoneType: None)]
DONE INLINING <code object inner at 0x7f7f5ddf8f50, file "/tmp/ipykernel_503/617960493.py", line 3>
TRACE POP_TOP None [ConstantVariable(NoneType: None)]
TRACE starts_line /tmp/ipykernel_503/617960493.py:9 in fn
        inner(1)
TRACE LOAD_FAST inner []
TRACE LOAD_CONST 1 [NestedUserFunctionVariable()]
TRACE CALL_FUNCTION 1 [NestedUserFunctionVariable(), ConstantVariable(int: 1)]
INLINING <code object inner at 0x7f7f5ddf8f50, file "/tmp/ipykernel_503/617960493.py", line 3>, inlined according trace_rules.lookup inlined by default
TRACE starts_line /tmp/ipykernel_503/617960493.py:5 in inner (inline depth: 1)
            x = x + 1
TRACE LOAD_DEREF x []
TRACE LOAD_CONST 1 [TensorVariable()]
TRACE BINARY_ADD None [TensorVariable(), ConstantVariable(int: 1)]
TRACE STORE_DEREF x [TensorVariable()]
TRACE starts_line /tmp/ipykernel_503/617960493.py:6 in inner (inline depth: 1)
            if i == 3:
TRACE LOAD_FAST i []
TRACE LOAD_CONST 3 [ConstantVariable(int: 1)]
TRACE COMPARE_OP == [ConstantVariable(int: 1), ConstantVariable(int: 3)]
TRACE POP_JUMP_IF_FALSE 30 [ConstantVariable(bool: False)]
TRACE LOAD_CONST None []
TRACE RETURN_VALUE None [ConstantVariable(NoneType: None)]
DONE INLINING <code object inner at 0x7f7f5ddf8f50, file "/tmp/ipykernel_503/617960493.py", line 3>
TRACE POP_TOP None [ConstantVariable(NoneType: None)]
TRACE starts_line /tmp/ipykernel_503/617960493.py:10 in fn
        inner(2)
TRACE LOAD_FAST inner []
TRACE LOAD_CONST 2 [NestedUserFunctionVariable()]
TRACE CALL_FUNCTION 1 [NestedUserFunctionVariable(), ConstantVariable(int: 2)]
INLINING <code object inner at 0x7f7f5ddf8f50, file "/tmp/ipykernel_503/617960493.py", line 3>, inlined according trace_rules.lookup inlined by default
TRACE starts_line /tmp/ipykernel_503/617960493.py:5 in inner (inline depth: 1)
            x = x + 1
TRACE LOAD_DEREF x []
TRACE LOAD_CONST 1 [TensorVariable()]
TRACE BINARY_ADD None [TensorVariable(), ConstantVariable(int: 1)]
TRACE STORE_DEREF x [TensorVariable()]
TRACE starts_line /tmp/ipykernel_503/617960493.py:6 in inner (inline depth: 1)
            if i == 3:
TRACE LOAD_FAST i []
TRACE LOAD_CONST 3 [ConstantVariable(int: 2)]
TRACE COMPARE_OP == [ConstantVariable(int: 2), ConstantVariable(int: 3)]
TRACE POP_JUMP_IF_FALSE 30 [ConstantVariable(bool: False)]
TRACE LOAD_CONST None []
TRACE RETURN_VALUE None [ConstantVariable(NoneType: None)]
DONE INLINING <code object inner at 0x7f7f5ddf8f50, file "/tmp/ipykernel_503/617960493.py", line 3>
TRACE POP_TOP None [ConstantVariable(NoneType: None)]
TRACE starts_line /tmp/ipykernel_503/617960493.py:11 in fn
        inner(3)
TRACE LOAD_FAST inner []
TRACE LOAD_CONST 3 [NestedUserFunctionVariable()]
TRACE CALL_FUNCTION 1 [NestedUserFunctionVariable(), ConstantVariable(int: 3)]
CALL_FUNCTION triggered compile
COMPILING GRAPH due to GraphCompileReason(reason='Call to `torch._dynamo.graph_break()`\n  Explanation: User-inserted graph break. Message: None\n  Hint: Remove the `torch._dynamo.graph_break()` call.\n\n  Developer debug context: Called `torch._dynamo.graph_break()` with args `[]`, kwargs `{}`\n\n For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb0025.html', user_stack=[<FrameSummary file /tmp/ipykernel_503/617960493.py, line 11 in fn>, <FrameSummary file /tmp/ipykernel_503/617960493.py, line 7 in inner>], graph_break=True)
Mutating object of type <unknown type> (source: x)

      File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
        return _run_code(code, main_globals, None,
      File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
        exec(code, run_globals)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
        app.launch_new_instance()
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
        app.start()
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
        self.io_loop.start()
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
        self.asyncio_loop.run_forever()
      File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
        self._run_once()
      File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
        handle._run()
      File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
        self._context.run(self._callback, *self._args)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
        return await f(*args, **kwargs)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
        await self.dispatch_shell(msg, subshell_id=subshell_id)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
        await result
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
        await super().execute_request(stream, ident, parent)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
        reply_content = await reply_content
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
        res = shell.run_cell(
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
        return super().run_cell(*args, **kwargs)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
        result = self._run_cell(
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
        result = runner(coro)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
        coro.send(None)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
        has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
        if await self.run_code(code, result, async_=asy):
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
        exec(code_obj, self.user_global_ns, self.user_ns)
      File "/tmp/ipykernel_503/617960493.py", line 14, in <module>
        fn(torch.randn(3))

    ********

      File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
        return _run_code(code, main_globals, None,
      File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
        exec(code, run_globals)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
        app.launch_new_instance()
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
        app.start()
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
        self.io_loop.start()
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
        self.asyncio_loop.run_forever()
      File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
        self._run_once()
      File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
        handle._run()
      File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
        self._context.run(self._callback, *self._args)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
        return await f(*args, **kwargs)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
        await self.dispatch_shell(msg, subshell_id=subshell_id)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
        await result
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
        await super().execute_request(stream, ident, parent)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
        reply_content = await reply_content
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
        res = shell.run_cell(
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
        return super().run_cell(*args, **kwargs)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
        result = self._run_cell(
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
        result = runner(coro)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
        coro.send(None)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
        has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
        if await self.run_code(code, result, async_=asy):
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
        exec(code_obj, self.user_global_ns, self.user_ns)
      File "/tmp/ipykernel_503/617960493.py", line 14, in <module>
        fn(torch.randn(3))
      File "/tmp/ipykernel_503/617960493.py", line 8, in fn
        inner(0)
      File "/tmp/ipykernel_503/617960493.py", line 5, in inner
        x = x + 1

    ********

      File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
        return _run_code(code, main_globals, None,
      File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
        exec(code, run_globals)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
        app.launch_new_instance()
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
        app.start()
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
        self.io_loop.start()
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
        self.asyncio_loop.run_forever()
      File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
        self._run_once()
      File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
        handle._run()
      File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
        self._context.run(self._callback, *self._args)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
        return await f(*args, **kwargs)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
        await self.dispatch_shell(msg, subshell_id=subshell_id)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
        await result
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
        await super().execute_request(stream, ident, parent)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
        reply_content = await reply_content
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
        res = shell.run_cell(
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
        return super().run_cell(*args, **kwargs)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
        result = self._run_cell(
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
        result = runner(coro)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
        coro.send(None)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
        has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
        if await self.run_code(code, result, async_=asy):
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
        exec(code_obj, self.user_global_ns, self.user_ns)
      File "/tmp/ipykernel_503/617960493.py", line 14, in <module>
        fn(torch.randn(3))
      File "/tmp/ipykernel_503/617960493.py", line 9, in fn
        inner(1)
      File "/tmp/ipykernel_503/617960493.py", line 5, in inner
        x = x + 1

    ********

      File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
        return _run_code(code, main_globals, None,
      File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
        exec(code, run_globals)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
        app.launch_new_instance()
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
        app.start()
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
        self.io_loop.start()
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
        self.asyncio_loop.run_forever()
      File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
        self._run_once()
      File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
        handle._run()
      File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
        self._context.run(self._callback, *self._args)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
        return await f(*args, **kwargs)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
        await self.dispatch_shell(msg, subshell_id=subshell_id)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
        await result
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
        await super().execute_request(stream, ident, parent)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
        reply_content = await reply_content
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
        res = shell.run_cell(
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
        return super().run_cell(*args, **kwargs)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
        result = self._run_cell(
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
        result = runner(coro)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
        coro.send(None)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
        has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
        if await self.run_code(code, result, async_=asy):
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
        exec(code_obj, self.user_global_ns, self.user_ns)
      File "/tmp/ipykernel_503/617960493.py", line 14, in <module>
        fn(torch.randn(3))
      File "/tmp/ipykernel_503/617960493.py", line 10, in fn
        inner(2)
      File "/tmp/ipykernel_503/617960493.py", line 5, in inner
        x = x + 1
TRACED GRAPH
 ===== __compiled_fn_8_31f13eb3_5f4f_4e09_b5da_eb0237e4e84d =====
 /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/fx/_lazy_graph_module.py class GraphModule(torch.nn.Module):
    def forward(self, L_x_: "f32[3][1]cpu"):
        l_x_ = L_x_

        # File: /tmp/ipykernel_503/617960493.py:5 in inner, code: x = x + 1
        x: "f32[3][1]cpu" = l_x_ + 1;  l_x_ = None
        x_1: "f32[3][1]cpu" = x + 1;  x = None
        x_2: "f32[3][1]cpu" = x_1 + 1;  x_1 = None
        return (x_2,)
Step 2: calling compiler function eager
Step 2: done compiler function eager
produce_guards
track_symint L['x'].size()[0] 3 None
track_symint L['x'].stride()[0] 1 None
track_symint L['x'].storage_offset() 0 None
Skipping guard L['x'].size()[0] == 3
Skipping guard L['x'].stride()[0] == 1
Skipping guard L['x'].storage_offset() == 0
GUARDS:

TREE_GUARD_MANAGER:
+- RootGuardManager
| +- LAMBDA_GUARD: torch._functorch.aot_autograd.utils.top_saved_tensors_hooks ids == None  # _dynamo/output_graph.py:866 in init_ambient_guards
| +- GLOBAL_STATE: ___check_global_state() against {"allow_bf16_reduce":0,"allow_fp16_reduce":0,"allow_tf32":false,"autocast_state":{"cached_enabled":true,"dtype":[15,5,5,15,5,5,15,15,5,5],"enabled":[false,false,false,false,false,false,false,false,false,false]},"default_dtype":6,"deterministic_algorithms":false,"deterministic_algorithms_warn_only":false,"grad_mode":true,"num_threads":4,"torch_function":true,"torch_function_all_disabled":false}
| +- TORCH_FUNCTION_MODE_STACK: ___check_torch_function_mode_stack()
| +- DEFAULT_DEVICE: utils_device.CURRENT_DEVICE == None                           # _dynamo/output_graph.py:853 in init_ambient_guards
| +- GuardManager: source=L['x'], accessed_by=FrameLocalsGuardAccessor(key='x', framelocals_idx=2), type=<class 'torch.Tensor'>, tag_safe=(True, False)
| | +- TENSOR_MATCH: check_tensor(L['x'], Tensor, DispatchKeySet(CPU, BackendSelect, ADInplaceOrView, AutogradCPU), torch.float32, device=None, requires_grad=False, size=[3], stride=[1])  # inner(0)  # mp/ipykernel_503/617960493.py:8 in fn
| | +- NO_HASATTR: hasattr(L['x'], '_dynamo_dynamic_indices') == False           # inner(0)  # mp/ipykernel_503/617960493.py:8 in fn
Guard eval latency = 286.70 us
put_code_state: no cache key, skipping
run_gc_after_compile: running gc
skipping: _create_nested_fn (reason: in skipfiles, file: /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/variables/functions.py)
torchdynamo start compiling inner /tmp/ipykernel_503/617960493.py:3, stack (elided 5 frames):
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
    app.launch_new_instance()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
    app.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
    self.io_loop.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
    self.asyncio_loop.run_forever()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
    self._run_once()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
    handle._run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
    self._context.run(self._callback, *self._args)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
    return await f(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
    await self.dispatch_shell(msg, subshell_id=subshell_id)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
    await result
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
    await super().execute_request(stream, ident, parent)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
    reply_content = await reply_content
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
    res = shell.run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
    return super().run_cell(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
    result = self._run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
    result = runner(coro)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
    coro.send(None)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
    if await self.run_code(code, result, async_=asy):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "/tmp/ipykernel_503/617960493.py", line 14, in <module>
    fn(torch.randn(3))
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 992, in compile_wrapper
    return fn(*args, **kwargs)
Step 1: torchdynamo start tracing inner /tmp/ipykernel_503/617960493.py:3
create_env
TRACE starts_line /tmp/ipykernel_503/617960493.py:5 in inner
            x = x + 1
TRACE LOAD_DEREF x []
TRACE LOAD_CONST 1 [LazyVariableTracker(unrealized: <class 'torch.Tensor'>)]
TRACE BINARY_ADD None [LazyVariableTracker(unrealized: <class 'torch.Tensor'>), ConstantVariable(int: 1)]
wrap_to_fake L['x'] (3,) StatefulSymbolicContext(dynamic_sizes=[<DimDynamic.STATIC: 2>], dynamic_strides=[<DimDynamic.INFER_STRIDE: 4>], constraint_sizes=[None], constraint_strides=[None], specialize_on=[[]], view_base_context=None, tensor_source=LocalSource(local_name='x', is_input=False, dynamism=None, is_derefed_cell_contents=True), shape_env_to_source_to_symbol_cache={}) <class 'torch.Tensor'>
create_graph_input L_x_ L['x'] FakeTensor(..., size=(3,)) at debug_level 0 before=False
TRACE STORE_DEREF x [TensorVariable()]
TRACE starts_line /tmp/ipykernel_503/617960493.py:6 in inner
            if i == 3:
TRACE LOAD_FAST i []
TRACE LOAD_CONST 3 [LazyVariableTracker(unrealized: <class 'int'>)]
TRACE COMPARE_OP == [LazyVariableTracker(unrealized: <class 'int'>), ConstantVariable(int: 3)]
TRACE POP_JUMP_IF_FALSE 30 [ConstantVariable(bool: True)]
TRACE starts_line /tmp/ipykernel_503/617960493.py:7 in inner
                torch._dynamo.graph_break()
TRACE LOAD_GLOBAL torch []
TRACE LOAD_ATTR _dynamo [LazyVariableTracker(unrealized: <class 'module'>)]
TRACE LOAD_ATTR graph_break [LazyVariableTracker(unrealized: <class 'module'>)]
TRACE CALL_FUNCTION 0 [LazyVariableTracker(unrealized: <class 'function'>)]
Graph break (user stack suppressed due to duplicate graph break) in user code at /tmp/ipykernel_503/617960493.py:7
Graph Break Reason: Encountered graph break when attempting to trace CALL_FUNCTION: a call to a regular function, e.g. f(x, y):

Call to `torch._dynamo.graph_break()`
  Explanation: User-inserted graph break. Message: None
  Hint: Remove the `torch._dynamo.graph_break()` call.

  Developer debug context: Called `torch._dynamo.graph_break()` with args `[]`, kwargs `{}`

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb0025.html
Restarting analysis due to _dynamo/symbolic_convert.py:252 in fail_and_restart_analysis
Step 1: torchdynamo start tracing inner /tmp/ipykernel_503/617960493.py:3
create_env
TRACE starts_line /tmp/ipykernel_503/617960493.py:5 in inner
            x = x + 1
TRACE LOAD_DEREF x []
TRACE LOAD_CONST 1 [LazyVariableTracker(unrealized: <class 'torch.Tensor'>)]
TRACE BINARY_ADD None [LazyVariableTracker(unrealized: <class 'torch.Tensor'>), ConstantVariable(int: 1)]
wrap_to_fake L['x'] (3,) StatefulSymbolicContext(dynamic_sizes=[<DimDynamic.STATIC: 2>], dynamic_strides=[<DimDynamic.INFER_STRIDE: 4>], constraint_sizes=[None], constraint_strides=[None], specialize_on=[[]], view_base_context=None, tensor_source=LocalSource(local_name='x', is_input=False, dynamism=None, is_derefed_cell_contents=True), shape_env_to_source_to_symbol_cache={}) <class 'torch.Tensor'>
create_graph_input L_x_ L['x'] FakeTensor(..., size=(3,)) at debug_level 0 before=False
TRACE STORE_DEREF x [TensorVariable()]
TRACE starts_line /tmp/ipykernel_503/617960493.py:6 in inner
            if i == 3:
TRACE LOAD_FAST i []
TRACE LOAD_CONST 3 [LazyVariableTracker(unrealized: <class 'int'>)]
TRACE COMPARE_OP == [LazyVariableTracker(unrealized: <class 'int'>), ConstantVariable(int: 3)]
TRACE POP_JUMP_IF_FALSE 30 [ConstantVariable(bool: True)]
TRACE starts_line /tmp/ipykernel_503/617960493.py:7 in inner
                torch._dynamo.graph_break()
TRACE LOAD_GLOBAL torch []
TRACE LOAD_ATTR _dynamo [LazyVariableTracker(unrealized: <class 'module'>)]
TRACE LOAD_ATTR graph_break [LazyVariableTracker(unrealized: <class 'module'>)]
TRACE CALL_FUNCTION 0 [LazyVariableTracker(unrealized: <class 'function'>)]
CALL_FUNCTION triggered compile
COMPILING GRAPH due to GraphCompileReason(reason='Call to `torch._dynamo.graph_break()`\n  Explanation: User-inserted graph break. Message: None\n  Hint: Remove the `torch._dynamo.graph_break()` call.\n\n  Developer debug context: Called `torch._dynamo.graph_break()` with args `[]`, kwargs `{}`\n\n For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb0025.html', user_stack=[<FrameSummary file /tmp/ipykernel_503/617960493.py, line 7 in inner>], graph_break=True)
Mutating object of type <unknown type> (source: x)

      File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
        return _run_code(code, main_globals, None,
      File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
        exec(code, run_globals)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
        app.launch_new_instance()
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
        app.start()
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
        self.io_loop.start()
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
        self.asyncio_loop.run_forever()
      File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
        self._run_once()
      File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
        handle._run()
      File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
        self._context.run(self._callback, *self._args)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
        return await f(*args, **kwargs)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
        await self.dispatch_shell(msg, subshell_id=subshell_id)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
        await result
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
        await super().execute_request(stream, ident, parent)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
        reply_content = await reply_content
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
        res = shell.run_cell(
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
        return super().run_cell(*args, **kwargs)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
        result = self._run_cell(
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
        result = runner(coro)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
        coro.send(None)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
        has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
        if await self.run_code(code, result, async_=asy):
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
        exec(code_obj, self.user_global_ns, self.user_ns)
      File "/tmp/ipykernel_503/617960493.py", line 14, in <module>
        fn(torch.randn(3))
      File "/tmp/ipykernel_503/617960493.py", line 11, in fn
        inner(3)
      File "/tmp/ipykernel_503/617960493.py", line 5, in inner
        x = x + 1
TRACED GRAPH
 ===== __compiled_fn_12_d7bcae16_3291_461b_85c6_2546051de602 =====
 /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/fx/_lazy_graph_module.py class GraphModule(torch.nn.Module):
    def forward(self, L_x_: "f32[3][1]cpu"):
        l_x_ = L_x_

        # File: /tmp/ipykernel_503/617960493.py:5 in inner, code: x = x + 1
        x: "f32[3][1]cpu" = l_x_ + 1;  l_x_ = None
        return (x,)
Step 2: calling compiler function eager
Step 2: done compiler function eager
produce_guards
track_symint L['x'].size()[0] 3 None
track_symint L['x'].stride()[0] 1 None
track_symint L['x'].storage_offset() 0 None
Skipping guard L['x'].size()[0] == 3
Skipping guard L['x'].stride()[0] == 1
Skipping guard L['x'].storage_offset() == 0
GUARDS:

TREE_GUARD_MANAGER:
+- RootGuardManager
| +- LAMBDA_GUARD: torch._functorch.aot_autograd.utils.top_saved_tensors_hooks ids == None  # _dynamo/output_graph.py:866 in init_ambient_guards
| +- GLOBAL_STATE: ___check_global_state() against {"allow_bf16_reduce":0,"allow_fp16_reduce":0,"allow_tf32":false,"autocast_state":{"cached_enabled":true,"dtype":[15,5,5,15,5,5,15,15,5,5],"enabled":[false,false,false,false,false,false,false,false,false,false]},"default_dtype":6,"deterministic_algorithms":false,"deterministic_algorithms_warn_only":false,"grad_mode":true,"num_threads":4,"torch_function":true,"torch_function_all_disabled":false}
| +- TORCH_FUNCTION_MODE_STACK: ___check_torch_function_mode_stack()
| +- DEFAULT_DEVICE: utils_device.CURRENT_DEVICE == None                           # _dynamo/output_graph.py:853 in init_ambient_guards
| +- GuardManager: source=L['i'], accessed_by=FrameLocalsGuardAccessor(key='i', framelocals_idx=0), type=<class 'int'>, tag_safe=(True, False)
| | +- EQUALS_MATCH: L['i'] == 3                                                   # if i == 3:  # mp/ipykernel_503/617960493.py:6 in inner
| +- GuardManager: source=L['x'], accessed_by=FrameLocalsGuardAccessor(key='x', framelocals_idx=1), type=<class 'torch.Tensor'>, tag_safe=(True, False)
| | +- TENSOR_MATCH: check_tensor(L['x'], Tensor, DispatchKeySet(CPU, BackendSelect, ADInplaceOrView, AutogradCPU), torch.float32, device=None, requires_grad=False, size=[3], stride=[1])  # x = x + 1  # mp/ipykernel_503/617960493.py:5 in inner
| | +- NO_HASATTR: hasattr(L['x'], '_dynamo_dynamic_indices') == False           # x = x + 1  # mp/ipykernel_503/617960493.py:5 in inner
| +- GuardManager: source=G, accessed_by=GlobalsGuardAccessor, type=<class 'dict'>, tag_safe=(False, False)
| | +- GuardManager: source=G['torch'], accessed_by=DictGetItemGuardAccessor('torch'), type=<class 'module'>, tag_safe=(False, False)
| | | +- ID_MATCH: ___check_obj_id(G['torch'], 140186730360128), type=<module 'torch' from '/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/__init__.py'>  # torch._dynamo.graph_break()  # mp/ipykernel_503/617960493.py:7 in inner
| | | +- GuardManager: source=G['torch']._dynamo, accessed_by=GetAttrGuardAccessor(_dynamo), type=<class 'module'>, tag_safe=(False, False)
| | | | +- ID_MATCH: ___check_obj_id(G['torch']._dynamo, 140185246904016), type=<module 'torch._dynamo' from '/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/__init__.py'>  # torch._dynamo.graph_break()  # mp/ipykernel_503/617960493.py:7 in inner
| | | | +- GuardManager: source=G['torch']._dynamo.graph_break, accessed_by=GetAttrGuardAccessor(graph_break), type=<class 'function'>, tag_safe=(True, False)
| | | | | +- GuardManager: source=G['torch']._dynamo.graph_break.__code__, accessed_by=CodeGuardAccessor, type=<class 'code'>, tag_safe=(True, False)
| | | | | | +- ID_MATCH: ___check_obj_id(G['torch']._dynamo.graph_break.__code__, 140185053639792), type=<code object graph_break at 0x7f7f60533470, file "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/decorators.py", line 290>  # torch._dynamo.graph_break()  # mp/ipykernel_503/617960493.py:7 in inner
Guard eval latency = 735.95 us
put_code_state: no cache key, skipping
run_gc_after_compile: running gc
torchdynamo start compiling torch_dynamo_resume_in_inner_at_7 /tmp/ipykernel_503/617960493.py:7, stack (elided 5 frames):
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
    app.launch_new_instance()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
    app.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
    self.io_loop.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
    self.asyncio_loop.run_forever()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
    self._run_once()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
    handle._run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
    self._context.run(self._callback, *self._args)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
    return await f(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
    await self.dispatch_shell(msg, subshell_id=subshell_id)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
    await result
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
    await super().execute_request(stream, ident, parent)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
    reply_content = await reply_content
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
    res = shell.run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
    return super().run_cell(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
    result = self._run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
    result = runner(coro)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
    coro.send(None)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
    if await self.run_code(code, result, async_=asy):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "/tmp/ipykernel_503/617960493.py", line 14, in <module>
    fn(torch.randn(3))
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 992, in compile_wrapper
    return fn(*args, **kwargs)
  File "/tmp/ipykernel_503/617960493.py", line 11, in fn
    inner(3)
Step 1: torchdynamo start tracing torch_dynamo_resume_in_inner_at_7 /tmp/ipykernel_503/617960493.py:7
create_env
TRACE starts_line /tmp/ipykernel_503/617960493.py:7 in torch_dynamo_resume_in_inner_at_7
                torch._dynamo.graph_break()
TRACE LOAD_CONST True []
TRACE STORE_FAST __is_tracing_resume_prologue [ConstantVariable(bool: True)]
TRACE LOAD_FAST ___stack0 []
TRACE LOAD_CONST False [LazyVariableTracker(unrealized: <class 'NoneType'>)]
TRACE STORE_FAST __is_tracing_resume_prologue [LazyVariableTracker(unrealized: <class 'NoneType'>), ConstantVariable(bool: False)]
TRACE JUMP_ABSOLUTE 36 [LazyVariableTracker(unrealized: <class 'NoneType'>)]
TRACE POP_TOP None [LazyVariableTracker(unrealized: <class 'NoneType'>)]
TRACE LOAD_CONST None []
TRACE RETURN_VALUE None [ConstantVariable(NoneType: None)]
Received signal to skip frame (without graph break): No ops traced for the FX graph. `torch.compile` will skip the frame and fall back to eager.
Frame info: torch_dynamo_resume_in_inner_at_7 (/tmp/ipykernel_503/617960493.py line 7) torch_dynamo_resume_in_inner_at_7                 /tmp/ipykernel_503/617960493.py 7
put_code_state: no cache key, skipping
run_gc_after_compile: running gc
torchdynamo start compiling torch_dynamo_resume_in_fn_at_11 /tmp/ipykernel_503/617960493.py:11, stack (elided 5 frames):
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
    app.launch_new_instance()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
    app.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
    self.io_loop.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
    self.asyncio_loop.run_forever()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
    self._run_once()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
    handle._run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
    self._context.run(self._callback, *self._args)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
    return await f(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
    await self.dispatch_shell(msg, subshell_id=subshell_id)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
    await result
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
    await super().execute_request(stream, ident, parent)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
    reply_content = await reply_content
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
    res = shell.run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
    return super().run_cell(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
    result = self._run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
    result = runner(coro)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
    coro.send(None)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
    if await self.run_code(code, result, async_=asy):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "/tmp/ipykernel_503/617960493.py", line 14, in <module>
    fn(torch.randn(3))
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 992, in compile_wrapper
    return fn(*args, **kwargs)
Step 1: torchdynamo start tracing torch_dynamo_resume_in_fn_at_11 /tmp/ipykernel_503/617960493.py:11
create_env
TRACE starts_line /tmp/ipykernel_503/617960493.py:11 in torch_dynamo_resume_in_fn_at_11
        inner(3)
TRACE LOAD_CONST True []
TRACE STORE_FAST __is_tracing_resume_prologue [ConstantVariable(bool: True)]
TRACE LOAD_FAST ___stack0 []
TRACE LOAD_CONST False [LazyVariableTracker(unrealized: <class 'NoneType'>)]
TRACE STORE_FAST __is_tracing_resume_prologue [LazyVariableTracker(unrealized: <class 'NoneType'>), ConstantVariable(bool: False)]
TRACE JUMP_ABSOLUTE 54 [LazyVariableTracker(unrealized: <class 'NoneType'>)]
TRACE POP_TOP None [LazyVariableTracker(unrealized: <class 'NoneType'>)]
TRACE starts_line /tmp/ipykernel_503/617960493.py:12 in torch_dynamo_resume_in_fn_at_11
        inner(4)
TRACE LOAD_FAST inner []
TRACE LOAD_CONST 4 [LazyVariableTracker(unrealized: <class 'function'>)]
TRACE CALL_FUNCTION 1 [LazyVariableTracker(unrealized: <class 'function'>), ConstantVariable(int: 4)]
INLINING <code object inner at 0x7f7f5ddf8f50, file "/tmp/ipykernel_503/617960493.py", line 3>, inlined according trace_rules.lookup inlined by default
TRACE starts_line /tmp/ipykernel_503/617960493.py:5 in inner (inline depth: 1)
            x = x + 1
TRACE LOAD_DEREF x []
TRACE LOAD_CONST 1 [LazyVariableTracker(unrealized: <class 'torch.Tensor'>)]
TRACE BINARY_ADD None [LazyVariableTracker(unrealized: <class 'torch.Tensor'>), ConstantVariable(int: 1)]
wrap_to_fake L['x'] (3,) StatefulSymbolicContext(dynamic_sizes=[<DimDynamic.STATIC: 2>], dynamic_strides=[<DimDynamic.INFER_STRIDE: 4>], constraint_sizes=[None], constraint_strides=[None], specialize_on=[[]], view_base_context=None, tensor_source=LocalSource(local_name='x', is_input=False, dynamism=None, is_derefed_cell_contents=True), shape_env_to_source_to_symbol_cache={}) <class 'torch.Tensor'>
create_graph_input L_x_ L['x'] FakeTensor(..., size=(3,)) at debug_level 0 before=False
TRACE STORE_DEREF x [TensorVariable()]
TRACE starts_line /tmp/ipykernel_503/617960493.py:6 in inner (inline depth: 1)
            if i == 3:
TRACE LOAD_FAST i []
TRACE LOAD_CONST 3 [ConstantVariable(int: 4)]
TRACE COMPARE_OP == [ConstantVariable(int: 4), ConstantVariable(int: 3)]
TRACE POP_JUMP_IF_FALSE 30 [ConstantVariable(bool: False)]
TRACE LOAD_CONST None []
TRACE RETURN_VALUE None [ConstantVariable(NoneType: None)]
DONE INLINING <code object inner at 0x7f7f5ddf8f50, file "/tmp/ipykernel_503/617960493.py", line 3>
TRACE POP_TOP None [ConstantVariable(NoneType: None)]
TRACE starts_line /tmp/ipykernel_503/617960493.py:13 in torch_dynamo_resume_in_fn_at_11
        return x
TRACE LOAD_DEREF x []
TRACE RETURN_VALUE None [TensorVariable()]
Step 1: torchdynamo done tracing torch_dynamo_resume_in_fn_at_11 (RETURN_VALUE)
return triggered compile
COMPILING GRAPH due to GraphCompileReason(reason='return_value', user_stack=[<FrameSummary file /tmp/ipykernel_503/617960493.py, line 13 in torch_dynamo_resume_in_fn_at_11>], graph_break=False)
Mutating object of type <unknown type> (source: x)

      File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
        return _run_code(code, main_globals, None,
      File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
        exec(code, run_globals)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
        app.launch_new_instance()
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
        app.start()
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
        self.io_loop.start()
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
        self.asyncio_loop.run_forever()
      File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
        self._run_once()
      File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
        handle._run()
      File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
        self._context.run(self._callback, *self._args)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
        return await f(*args, **kwargs)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
        await self.dispatch_shell(msg, subshell_id=subshell_id)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
        await result
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
        await super().execute_request(stream, ident, parent)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
        reply_content = await reply_content
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
        res = shell.run_cell(
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
        return super().run_cell(*args, **kwargs)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
        result = self._run_cell(
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
        result = runner(coro)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
        coro.send(None)
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
        has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
        if await self.run_code(code, result, async_=asy):
      File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
        exec(code_obj, self.user_global_ns, self.user_ns)
      File "/tmp/ipykernel_503/617960493.py", line 14, in <module>
        fn(torch.randn(3))
      File "/tmp/ipykernel_503/617960493.py", line 12, in fn
        inner(4)
      File "/tmp/ipykernel_503/617960493.py", line 5, in inner
        x = x + 1
TRACED GRAPH
 ===== __compiled_fn_16_1ceb601f_76d2_41d3_82fe_cb093aa7991b =====
 /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/fx/_lazy_graph_module.py class GraphModule(torch.nn.Module):
    def forward(self, L_x_: "f32[3][1]cpu"):
        l_x_ = L_x_

        # File: /tmp/ipykernel_503/617960493.py:5 in inner, code: x = x + 1
        x: "f32[3][1]cpu" = l_x_ + 1;  l_x_ = None
        return (x,)
Step 2: calling compiler function eager
Step 2: done compiler function eager
produce_guards
track_symint L['x'].size()[0] 3 None
track_symint L['x'].stride()[0] 1 None
track_symint L['x'].storage_offset() 0 None
Skipping guard L['x'].size()[0] == 3
Skipping guard L['x'].stride()[0] == 1
Skipping guard L['x'].storage_offset() == 0
GUARDS:

TREE_GUARD_MANAGER:
+- RootGuardManager
| +- LAMBDA_GUARD: torch._functorch.aot_autograd.utils.top_saved_tensors_hooks ids == None  # _dynamo/output_graph.py:866 in init_ambient_guards
| +- GLOBAL_STATE: ___check_global_state() against {"allow_bf16_reduce":0,"allow_fp16_reduce":0,"allow_tf32":false,"autocast_state":{"cached_enabled":true,"dtype":[15,5,5,15,5,5,15,15,5,5],"enabled":[false,false,false,false,false,false,false,false,false,false]},"default_dtype":6,"deterministic_algorithms":false,"deterministic_algorithms_warn_only":false,"grad_mode":true,"num_threads":4,"torch_function":true,"torch_function_all_disabled":false}
| +- TORCH_FUNCTION_MODE_STACK: ___check_torch_function_mode_stack()
| +- DEFAULT_DEVICE: utils_device.CURRENT_DEVICE == None                           # _dynamo/output_graph.py:853 in init_ambient_guards
| +- GuardManager: source=L['x'], accessed_by=FrameLocalsGuardAccessor(key='x', framelocals_idx=6), type=<class 'torch.Tensor'>, tag_safe=(True, False)
| | +- TENSOR_MATCH: check_tensor(L['x'], Tensor, DispatchKeySet(CPU, BackendSelect, ADInplaceOrView, AutogradCPU), torch.float32, device=None, requires_grad=False, size=[3], stride=[1])  # x = x + 1  # mp/ipykernel_503/617960493.py:5 in inner
| | +- NO_HASATTR: hasattr(L['x'], '_dynamo_dynamic_indices') == False           # x = x + 1  # mp/ipykernel_503/617960493.py:5 in inner
| +- GuardManager: source=L['inner'], accessed_by=FrameLocalsGuardAccessor(key='inner', framelocals_idx=3), type=<class 'function'>, tag_safe=(True, False)
| | +- GuardManager: source=L['inner'].__code__, accessed_by=CodeGuardAccessor, type=<class 'code'>, tag_safe=(True, False)
| | | +- ID_MATCH: ___check_obj_id(L['inner'].__code__, 140185012506448), type=<code object inner at 0x7f7f5ddf8f50, file "/tmp/ipykernel_503/617960493.py", line 3>  # inner(4)  # mp/ipykernel_503/617960493.py:12 in torch_dynamo_resume_in_fn_at_11
Guard eval latency = 324.21 us
put_code_state: no cache key, skipping
run_gc_after_compile: running gc
skipping because no torch.* remove             /opt/conda/envs/py_3.10/lib/python3.10/weakref.py 370

tensor([7.3559, 3.6280, 4.5308])

In general, resolving the graph break causing the skip will also resolve the skip.

Graph Break in a Context Manager

Another common example of an unresumable graph break is a graph break in most context managers:

class CustomCtxManager:
    def __enter__(self):
        pass
    def __exit__(self, exc_type, exc_value, traceback):
        pass
@torch.compile
def fn(x):
    with CustomCtxManager():
        x = x + 1
        torch._dynamo.graph_break()
        return x + 1
fn(torch.randn(3))

torchdynamo start compiling fn /tmp/ipykernel_503/4148913404.py:6, stack (elided 5 frames):
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
    app.launch_new_instance()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
    app.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
    self.io_loop.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
    self.asyncio_loop.run_forever()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
    self._run_once()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
    handle._run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
    self._context.run(self._callback, *self._args)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
    return await f(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
    await self.dispatch_shell(msg, subshell_id=subshell_id)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
    await result
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
    await super().execute_request(stream, ident, parent)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
    reply_content = await reply_content
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
    res = shell.run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
    return super().run_cell(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
    result = self._run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
    result = runner(coro)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
    coro.send(None)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
    if await self.run_code(code, result, async_=asy):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "/tmp/ipykernel_503/4148913404.py", line 12, in <module>
    fn(torch.randn(3))
Step 1: torchdynamo start tracing fn /tmp/ipykernel_503/4148913404.py:6
create_env
TRACE starts_line /tmp/ipykernel_503/4148913404.py:8 in fn
        with CustomCtxManager():
TRACE LOAD_GLOBAL CustomCtxManager []
TRACE CALL_FUNCTION 0 [LazyVariableTracker(unrealized: <class 'type'>)]
TRACE SETUP_WITH 48 [GenericContextWrappingVariable(CustomCtxManager)]
INLINING <code object __enter__ at 0x7f7fc7f68a80, file "/tmp/ipykernel_503/4148913404.py", line 2>, inlined according trace_rules.lookup inlined by default
TRACE starts_line /tmp/ipykernel_503/4148913404.py:3 in __enter__ (inline depth: 1)
            pass
TRACE LOAD_CONST None []
TRACE RETURN_VALUE None [ConstantVariable(NoneType: None)]
DONE INLINING <code object __enter__ at 0x7f7fc7f68a80, file "/tmp/ipykernel_503/4148913404.py", line 2>
TRACE POP_TOP None [WithExitFunctionVariable(), ConstantVariable(NoneType: None)]
TRACE starts_line /tmp/ipykernel_503/4148913404.py:9 in fn
            x = x + 1
TRACE LOAD_FAST x [WithExitFunctionVariable()]
TRACE LOAD_CONST 1 [WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'torch.Tensor'>)]
TRACE BINARY_ADD None [WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'torch.Tensor'>), ConstantVariable(int: 1)]
wrap_to_fake L['x'] (3,) StatefulSymbolicContext(dynamic_sizes=[<DimDynamic.STATIC: 2>], dynamic_strides=[<DimDynamic.INFER_STRIDE: 4>], constraint_sizes=[None], constraint_strides=[None], specialize_on=[[]], view_base_context=None, tensor_source=LocalSource(local_name='x', is_input=True, dynamism=None, is_derefed_cell_contents=False), shape_env_to_source_to_symbol_cache={}) <class 'torch.Tensor'>
create_graph_input L_x_ L['x'] FakeTensor(..., size=(3,)) at debug_level 0 before=False
TRACE STORE_FAST x [WithExitFunctionVariable(), TensorVariable()]
TRACE starts_line /tmp/ipykernel_503/4148913404.py:10 in fn
            torch._dynamo.graph_break()
TRACE LOAD_GLOBAL torch [WithExitFunctionVariable()]
TRACE LOAD_ATTR _dynamo [WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'module'>)]
TRACE LOAD_ATTR graph_break [WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'module'>)]
TRACE CALL_FUNCTION 0 [WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'function'>)]
empty checkpoint - cannot resume from graph break
Graph break in user code at /tmp/ipykernel_503/4148913404.py:10
Graph Break Reason: Failed to handle graph break gracefully. Skipping the function and falling back to eager. Graph break encountered:

Call to `torch._dynamo.graph_break()`
  Explanation: User-inserted graph break. Message: None
  Hint: Remove the `torch._dynamo.graph_break()` call.

  Developer debug context: Called `torch._dynamo.graph_break()` with args `[]`, kwargs `{}`

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb0025.html

*** While handling this graph break, another graph break occurred: ***

Graph break under GenericContextWrappingVariable
  Explanation: Attempted to graph break in an active context manager(s) that doesn't support graph breaking.
  Hint: Move the offending context manager(s) to outside the compiled region.
  Hint: This graph break may have been caused by an earlier graph break. Resolving the earlier graph break may resolve this one.

  Developer debug context: Active generic context managers: [GenericContextWrappingVariable(CustomCtxManager)]

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb0066.html

User code traceback:
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
    app.launch_new_instance()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
    app.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
    self.io_loop.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
    self.asyncio_loop.run_forever()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
    self._run_once()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
    handle._run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
    self._context.run(self._callback, *self._args)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
    return await f(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
    await self.dispatch_shell(msg, subshell_id=subshell_id)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
    await result
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
    await super().execute_request(stream, ident, parent)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
    reply_content = await reply_content
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
    res = shell.run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
    return super().run_cell(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
    result = self._run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
    result = runner(coro)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
    coro.send(None)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
    if await self.run_code(code, result, async_=asy):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "/tmp/ipykernel_503/4148913404.py", line 12, in <module>
    fn(torch.randn(3))
  File "/tmp/ipykernel_503/4148913404.py", line 10, in fn
    torch._dynamo.graph_break()
run_gc_after_compile: running gc
WON'T CONVERT fn /tmp/ipykernel_503/4148913404.py line 6 
due to: 
Traceback (most recent call last):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 888, in wrapper
    return inner_fn(self, inst)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 2605, in CALL_FUNCTION
    self.call_function(fn, args, {})
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1247, in call_function
    self.push(fn.call_function(self, args, kwargs))  # type: ignore[arg-type]
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/variables/lazy.py", line 433, in realize_and_forward
    return getattr(self.realize(), name)(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/variables/functions.py", line 1939, in call_function
    unimplemented(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/exc.py", line 634, in unimplemented
    raise Unsupported(msg, gb_type, skip_frame)
torch._dynamo.exc.Unsupported: Call to `torch._dynamo.graph_break()`
  Explanation: User-inserted graph break. Message: None
  Hint: Remove the `torch._dynamo.graph_break()` call.

  Developer debug context: Called `torch._dynamo.graph_break()` with args `[]`, kwargs `{}`

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb0025.html

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1975, in __call__
    result = self._inner_convert(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 715, in __call__
    result = _compile(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1772, in _compile
    guarded_code, tracer_output = compile_inner(code, one_graph, hooks)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_utils_internal.py", line 96, in wrapper_function
    return function(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1445, in compile_inner
    return _compile_inner(code, one_graph, hooks)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1479, in _compile_inner
    dynamo_output = compile_frame(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1353, in compile_frame
    bytecode, tracer_output = transform_code_object(code, transform)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/bytecode_transformation.py", line 1608, in transform_code_object
    tracer_output = transformations(instructions, code_options)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1325, in transform
    tracer_output = trace_frame(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 327, in _fn
    return fn(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 849, in trace_frame
    run_tracer()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 830, in run_tracer
    tracer.run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1675, in run
    while self.step():
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1340, in step
    self.dispatch_table[inst.opcode](self, inst)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 898, in wrapper
    unimplemented(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/exc.py", line 630, in unimplemented
    raise Unsupported(msg, gb_type, skip_frame, real_stack=past_real_stack)
torch._dynamo.exc.Unsupported: Call to `torch._dynamo.graph_break()`
  Explanation: User-inserted graph break. Message: None
  Hint: Remove the `torch._dynamo.graph_break()` call.

  Developer debug context: Called `torch._dynamo.graph_break()` with args `[]`, kwargs `{}`

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb0025.html

*** While handling this graph break, another graph break occurred: ***

Graph break under GenericContextWrappingVariable
  Explanation: Attempted to graph break in an active context manager(s) that doesn't support graph breaking.
  Hint: Move the offending context manager(s) to outside the compiled region.
  Hint: This graph break may have been caused by an earlier graph break. Resolving the earlier graph break may resolve this one.

  Developer debug context: Active generic context managers: [GenericContextWrappingVariable(CustomCtxManager)]

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb0066.html

from user code:
   File "/tmp/ipykernel_503/4148913404.py", line 10, in fn
    torch._dynamo.graph_break()

Set TORCHDYNAMO_VERBOSE=1 for the internal stack trace (please do this especially if you're reporting a bug to PyTorch). For even more developer context, set TORCH_LOGS="+dynamo"

Traceback (most recent call last):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 888, in wrapper
    return inner_fn(self, inst)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 2605, in CALL_FUNCTION
    self.call_function(fn, args, {})
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1247, in call_function
    self.push(fn.call_function(self, args, kwargs))  # type: ignore[arg-type]
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/variables/lazy.py", line 433, in realize_and_forward
    return getattr(self.realize(), name)(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/variables/functions.py", line 1939, in call_function
    unimplemented(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/exc.py", line 634, in unimplemented
    raise Unsupported(msg, gb_type, skip_frame)
torch._dynamo.exc.Unsupported: Call to `torch._dynamo.graph_break()`
  Explanation: User-inserted graph break. Message: None
  Hint: Remove the `torch._dynamo.graph_break()` call.

  Developer debug context: Called `torch._dynamo.graph_break()` with args `[]`, kwargs `{}`

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb0025.html

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1975, in __call__
    result = self._inner_convert(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 715, in __call__
    result = _compile(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1772, in _compile
    guarded_code, tracer_output = compile_inner(code, one_graph, hooks)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_utils_internal.py", line 96, in wrapper_function
    return function(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1445, in compile_inner
    return _compile_inner(code, one_graph, hooks)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1479, in _compile_inner
    dynamo_output = compile_frame(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1353, in compile_frame
    bytecode, tracer_output = transform_code_object(code, transform)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/bytecode_transformation.py", line 1608, in transform_code_object
    tracer_output = transformations(instructions, code_options)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1325, in transform
    tracer_output = trace_frame(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 327, in _fn
    return fn(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 849, in trace_frame
    run_tracer()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 830, in run_tracer
    tracer.run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1675, in run
    while self.step():
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1340, in step
    self.dispatch_table[inst.opcode](self, inst)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 898, in wrapper
    unimplemented(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/exc.py", line 630, in unimplemented
    raise Unsupported(msg, gb_type, skip_frame, real_stack=past_real_stack)
torch._dynamo.exc.Unsupported: Call to `torch._dynamo.graph_break()`
  Explanation: User-inserted graph break. Message: None
  Hint: Remove the `torch._dynamo.graph_break()` call.

  Developer debug context: Called `torch._dynamo.graph_break()` with args `[]`, kwargs `{}`

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb0025.html

*** While handling this graph break, another graph break occurred: ***

Graph break under GenericContextWrappingVariable
  Explanation: Attempted to graph break in an active context manager(s) that doesn't support graph breaking.
  Hint: Move the offending context manager(s) to outside the compiled region.
  Hint: This graph break may have been caused by an earlier graph break. Resolving the earlier graph break may resolve this one.

  Developer debug context: Active generic context managers: [GenericContextWrappingVariable(CustomCtxManager)]

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb0066.html

from user code:
   File "/tmp/ipykernel_503/4148913404.py", line 10, in fn
    torch._dynamo.graph_break()

Set TORCHDYNAMO_VERBOSE=1 for the internal stack trace (please do this especially if you're reporting a bug to PyTorch). For even more developer context, set TORCH_LOGS="+dynamo"
skipping because no torch.* __enter__             /tmp/ipykernel_503/4148913404.py 2
skipping because no torch.* __exit__             /tmp/ipykernel_503/4148913404.py 4

tensor([2.5895, 1.9549, 2.9373])

We can avoid skipping by moving the graph break outside of the context manager:

@torch.compile
def fn(x):
    with CustomCtxManager():
        x = x + 1
    torch._dynamo.graph_break()
    with CustomCtxManager():
        return x + 1
fn(torch.randn(3))

torchdynamo start compiling fn /tmp/ipykernel_503/2124425154.py:1, stack (elided 5 frames):
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
    app.launch_new_instance()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
    app.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
    self.io_loop.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
    self.asyncio_loop.run_forever()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
    self._run_once()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
    handle._run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
    self._context.run(self._callback, *self._args)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
    return await f(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
    await self.dispatch_shell(msg, subshell_id=subshell_id)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
    await result
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
    await super().execute_request(stream, ident, parent)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
    reply_content = await reply_content
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
    res = shell.run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
    return super().run_cell(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
    result = self._run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
    result = runner(coro)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
    coro.send(None)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
    if await self.run_code(code, result, async_=asy):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "/tmp/ipykernel_503/2124425154.py", line 8, in <module>
    fn(torch.randn(3))
Step 1: torchdynamo start tracing fn /tmp/ipykernel_503/2124425154.py:1
create_env
TRACE starts_line /tmp/ipykernel_503/2124425154.py:3 in fn
        with CustomCtxManager():
TRACE LOAD_GLOBAL CustomCtxManager []
TRACE CALL_FUNCTION 0 [LazyVariableTracker(unrealized: <class 'type'>)]
TRACE SETUP_WITH 30 [GenericContextWrappingVariable(CustomCtxManager)]
INLINING <code object __enter__ at 0x7f7fc7f68a80, file "/tmp/ipykernel_503/4148913404.py", line 2>, inlined according trace_rules.lookup inlined by default
TRACE starts_line /tmp/ipykernel_503/4148913404.py:3 in __enter__ (inline depth: 1)
            pass
TRACE LOAD_CONST None []
TRACE RETURN_VALUE None [ConstantVariable(NoneType: None)]
DONE INLINING <code object __enter__ at 0x7f7fc7f68a80, file "/tmp/ipykernel_503/4148913404.py", line 2>
TRACE POP_TOP None [WithExitFunctionVariable(), ConstantVariable(NoneType: None)]
TRACE starts_line /tmp/ipykernel_503/2124425154.py:4 in fn
            x = x + 1
TRACE LOAD_FAST x [WithExitFunctionVariable()]
TRACE LOAD_CONST 1 [WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'torch.Tensor'>)]
TRACE BINARY_ADD None [WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'torch.Tensor'>), ConstantVariable(int: 1)]
wrap_to_fake L['x'] (3,) StatefulSymbolicContext(dynamic_sizes=[<DimDynamic.STATIC: 2>], dynamic_strides=[<DimDynamic.INFER_STRIDE: 4>], constraint_sizes=[None], constraint_strides=[None], specialize_on=[[]], view_base_context=None, tensor_source=LocalSource(local_name='x', is_input=True, dynamism=None, is_derefed_cell_contents=False), shape_env_to_source_to_symbol_cache={}) <class 'torch.Tensor'>
create_graph_input L_x_ L['x'] FakeTensor(..., size=(3,)) at debug_level 0 before=False
TRACE STORE_FAST x [WithExitFunctionVariable(), TensorVariable()]
TRACE POP_BLOCK None [WithExitFunctionVariable()]
TRACE starts_line /tmp/ipykernel_503/2124425154.py:3 in fn
        with CustomCtxManager():
TRACE LOAD_CONST None [WithExitFunctionVariable()]
TRACE DUP_TOP None [WithExitFunctionVariable(), ConstantVariable(NoneType: None)]
TRACE DUP_TOP None [WithExitFunctionVariable(), ConstantVariable(NoneType: None), ConstantVariable(NoneType: None)]
TRACE CALL_FUNCTION 3 [WithExitFunctionVariable(), ConstantVariable(NoneType: None), ConstantVariable(NoneType: None), ConstantVariable(NoneType: None)]
INLINING <code object __exit__ at 0x7f7f5e1aae40, file "/tmp/ipykernel_503/4148913404.py", line 4>, inlined according trace_rules.lookup inlined by default
TRACE starts_line /tmp/ipykernel_503/4148913404.py:5 in __exit__ (inline depth: 1)
            pass
TRACE LOAD_CONST None []
TRACE RETURN_VALUE None [ConstantVariable(NoneType: None)]
DONE INLINING <code object __exit__ at 0x7f7f5e1aae40, file "/tmp/ipykernel_503/4148913404.py", line 4>
TRACE POP_TOP None [ConstantVariable(NoneType: None)]
TRACE JUMP_FORWARD 46 []
TRACE starts_line /tmp/ipykernel_503/2124425154.py:5 in fn
        torch._dynamo.graph_break()
TRACE LOAD_GLOBAL torch []
TRACE LOAD_ATTR _dynamo [LazyVariableTracker(unrealized: <class 'module'>)]
TRACE LOAD_ATTR graph_break [LazyVariableTracker(unrealized: <class 'module'>)]
TRACE CALL_FUNCTION 0 [LazyVariableTracker(unrealized: <class 'function'>)]
Graph break in user code at /tmp/ipykernel_503/2124425154.py:5
Graph Break Reason: Encountered graph break when attempting to trace CALL_FUNCTION: a call to a regular function, e.g. f(x, y):

Call to `torch._dynamo.graph_break()`
  Explanation: User-inserted graph break. Message: None
  Hint: Remove the `torch._dynamo.graph_break()` call.

  Developer debug context: Called `torch._dynamo.graph_break()` with args `[]`, kwargs `{}`

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb0025.html

User code traceback:
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
    app.launch_new_instance()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
    app.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
    self.io_loop.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
    self.asyncio_loop.run_forever()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
    self._run_once()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
    handle._run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
    self._context.run(self._callback, *self._args)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
    return await f(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
    await self.dispatch_shell(msg, subshell_id=subshell_id)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
    await result
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
    await super().execute_request(stream, ident, parent)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
    reply_content = await reply_content
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
    res = shell.run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
    return super().run_cell(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
    result = self._run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
    result = runner(coro)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
    coro.send(None)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
    if await self.run_code(code, result, async_=asy):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "/tmp/ipykernel_503/2124425154.py", line 8, in <module>
    fn(torch.randn(3))
  File "/tmp/ipykernel_503/2124425154.py", line 5, in fn
    torch._dynamo.graph_break()
Restarting analysis due to _dynamo/symbolic_convert.py:252 in fail_and_restart_analysis
Step 1: torchdynamo start tracing fn /tmp/ipykernel_503/2124425154.py:1
create_env
TRACE starts_line /tmp/ipykernel_503/2124425154.py:3 in fn
        with CustomCtxManager():
TRACE LOAD_GLOBAL CustomCtxManager []
TRACE CALL_FUNCTION 0 [LazyVariableTracker(unrealized: <class 'type'>)]
TRACE SETUP_WITH 30 [GenericContextWrappingVariable(CustomCtxManager)]
INLINING <code object __enter__ at 0x7f7fc7f68a80, file "/tmp/ipykernel_503/4148913404.py", line 2>, inlined according trace_rules.lookup inlined by default
TRACE starts_line /tmp/ipykernel_503/4148913404.py:3 in __enter__ (inline depth: 1)
            pass
TRACE LOAD_CONST None []
TRACE RETURN_VALUE None [ConstantVariable(NoneType: None)]
DONE INLINING <code object __enter__ at 0x7f7fc7f68a80, file "/tmp/ipykernel_503/4148913404.py", line 2>
TRACE POP_TOP None [WithExitFunctionVariable(), ConstantVariable(NoneType: None)]
TRACE starts_line /tmp/ipykernel_503/2124425154.py:4 in fn
            x = x + 1
TRACE LOAD_FAST x [WithExitFunctionVariable()]
TRACE LOAD_CONST 1 [WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'torch.Tensor'>)]
TRACE BINARY_ADD None [WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'torch.Tensor'>), ConstantVariable(int: 1)]
wrap_to_fake L['x'] (3,) StatefulSymbolicContext(dynamic_sizes=[<DimDynamic.STATIC: 2>], dynamic_strides=[<DimDynamic.INFER_STRIDE: 4>], constraint_sizes=[None], constraint_strides=[None], specialize_on=[[]], view_base_context=None, tensor_source=LocalSource(local_name='x', is_input=True, dynamism=None, is_derefed_cell_contents=False), shape_env_to_source_to_symbol_cache={}) <class 'torch.Tensor'>
create_graph_input L_x_ L['x'] FakeTensor(..., size=(3,)) at debug_level 0 before=False
TRACE STORE_FAST x [WithExitFunctionVariable(), TensorVariable()]
TRACE POP_BLOCK None [WithExitFunctionVariable()]
TRACE starts_line /tmp/ipykernel_503/2124425154.py:3 in fn
        with CustomCtxManager():
TRACE LOAD_CONST None [WithExitFunctionVariable()]
TRACE DUP_TOP None [WithExitFunctionVariable(), ConstantVariable(NoneType: None)]
TRACE DUP_TOP None [WithExitFunctionVariable(), ConstantVariable(NoneType: None), ConstantVariable(NoneType: None)]
TRACE CALL_FUNCTION 3 [WithExitFunctionVariable(), ConstantVariable(NoneType: None), ConstantVariable(NoneType: None), ConstantVariable(NoneType: None)]
INLINING <code object __exit__ at 0x7f7f5e1aae40, file "/tmp/ipykernel_503/4148913404.py", line 4>, inlined according trace_rules.lookup inlined by default
TRACE starts_line /tmp/ipykernel_503/4148913404.py:5 in __exit__ (inline depth: 1)
            pass
TRACE LOAD_CONST None []
TRACE RETURN_VALUE None [ConstantVariable(NoneType: None)]
DONE INLINING <code object __exit__ at 0x7f7f5e1aae40, file "/tmp/ipykernel_503/4148913404.py", line 4>
TRACE POP_TOP None [ConstantVariable(NoneType: None)]
TRACE JUMP_FORWARD 46 []
TRACE starts_line /tmp/ipykernel_503/2124425154.py:5 in fn
        torch._dynamo.graph_break()
TRACE LOAD_GLOBAL torch []
TRACE LOAD_ATTR _dynamo [LazyVariableTracker(unrealized: <class 'module'>)]
TRACE LOAD_ATTR graph_break [LazyVariableTracker(unrealized: <class 'module'>)]
TRACE CALL_FUNCTION 0 [LazyVariableTracker(unrealized: <class 'function'>)]
CALL_FUNCTION triggered compile
COMPILING GRAPH due to GraphCompileReason(reason='Call to `torch._dynamo.graph_break()`\n  Explanation: User-inserted graph break. Message: None\n  Hint: Remove the `torch._dynamo.graph_break()` call.\n\n  Developer debug context: Called `torch._dynamo.graph_break()` with args `[]`, kwargs `{}`\n\n For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb0025.html', user_stack=[<FrameSummary file /tmp/ipykernel_503/2124425154.py, line 5 in fn>], graph_break=True)
TRACED GRAPH
 ===== __compiled_fn_20_3135c6a8_1288_49db_afae_72f2dc6cd8f8 =====
 /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/fx/_lazy_graph_module.py class GraphModule(torch.nn.Module):
    def forward(self, L_x_: "f32[3][1]cpu"):
        l_x_ = L_x_

        # File: /tmp/ipykernel_503/2124425154.py:4 in fn, code: x = x + 1
        x: "f32[3][1]cpu" = l_x_ + 1;  l_x_ = None
        return (x,)
Step 2: calling compiler function eager
Step 2: done compiler function eager
produce_guards
track_symint L['x'].size()[0] 3 None
track_symint L['x'].stride()[0] 1 None
track_symint L['x'].storage_offset() 0 None
Skipping guard L['x'].size()[0] == 3
Skipping guard L['x'].stride()[0] == 1
Skipping guard L['x'].storage_offset() == 0
GUARDS:

TREE_GUARD_MANAGER:
+- RootGuardManager
| +- LAMBDA_GUARD: torch._functorch.aot_autograd.utils.top_saved_tensors_hooks ids == None  # _dynamo/output_graph.py:866 in init_ambient_guards
| +- GLOBAL_STATE: ___check_global_state() against {"allow_bf16_reduce":0,"allow_fp16_reduce":0,"allow_tf32":false,"autocast_state":{"cached_enabled":true,"dtype":[15,5,5,15,5,5,15,15,5,5],"enabled":[false,false,false,false,false,false,false,false,false,false]},"default_dtype":6,"deterministic_algorithms":false,"deterministic_algorithms_warn_only":false,"grad_mode":true,"num_threads":4,"torch_function":true,"torch_function_all_disabled":false}
| +- TORCH_FUNCTION_MODE_STACK: ___check_torch_function_mode_stack()
| +- DEFAULT_DEVICE: utils_device.CURRENT_DEVICE == None                           # _dynamo/output_graph.py:853 in init_ambient_guards
| +- GuardManager: source=L['x'], accessed_by=FrameLocalsGuardAccessor(key='x', framelocals_idx=0), type=<class 'torch.Tensor'>, tag_safe=(True, False)
| | +- TENSOR_MATCH: check_tensor(L['x'], Tensor, DispatchKeySet(CPU, BackendSelect, ADInplaceOrView, AutogradCPU), torch.float32, device=None, requires_grad=False, size=[3], stride=[1])  # x = x + 1  # mp/ipykernel_503/2124425154.py:4 in fn
| | +- NO_HASATTR: hasattr(L['x'], '_dynamo_dynamic_indices') == False           # x = x + 1  # mp/ipykernel_503/2124425154.py:4 in fn
| +- GuardManager: source=G, accessed_by=GlobalsGuardAccessor, type=<class 'dict'>, tag_safe=(False, False)
| | +- GuardManager: source=G['torch'], accessed_by=DictGetItemGuardAccessor('torch'), type=<class 'module'>, tag_safe=(False, False)
| | | +- ID_MATCH: ___check_obj_id(G['torch'], 140186730360128), type=<module 'torch' from '/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/__init__.py'>  # torch._dynamo.graph_break()  # mp/ipykernel_503/2124425154.py:5 in fn
| | | +- GuardManager: source=G['torch']._dynamo, accessed_by=GetAttrGuardAccessor(_dynamo), type=<class 'module'>, tag_safe=(False, False)
| | | | +- ID_MATCH: ___check_obj_id(G['torch']._dynamo, 140185246904016), type=<module 'torch._dynamo' from '/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/__init__.py'>  # torch._dynamo.graph_break()  # mp/ipykernel_503/2124425154.py:5 in fn
| | | | +- GuardManager: source=G['torch']._dynamo.graph_break, accessed_by=GetAttrGuardAccessor(graph_break), type=<class 'function'>, tag_safe=(True, False)
| | | | | +- GuardManager: source=G['torch']._dynamo.graph_break.__code__, accessed_by=CodeGuardAccessor, type=<class 'code'>, tag_safe=(True, False)
| | | | | | +- ID_MATCH: ___check_obj_id(G['torch']._dynamo.graph_break.__code__, 140185053639792), type=<code object graph_break at 0x7f7f60533470, file "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/decorators.py", line 290>  # torch._dynamo.graph_break()  # mp/ipykernel_503/2124425154.py:5 in fn
| | +- GuardManager: source=G['CustomCtxManager'], accessed_by=DictGetItemGuardAccessor('CustomCtxManager'), type=<class 'type'>, tag_safe=(True, False)
| | | +- ID_MATCH: ___check_obj_id(G['CustomCtxManager'], 93962434258400), type=<class '__main__.CustomCtxManager'>  # with CustomCtxManager():  # mp/ipykernel_503/2124425154.py:3 in fn
Guard eval latency = 444.68 us
put_code_state: no cache key, skipping
run_gc_after_compile: running gc
torchdynamo start compiling torch_dynamo_resume_in_fn_at_5 /tmp/ipykernel_503/2124425154.py:5, stack (elided 5 frames):
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
    app.launch_new_instance()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
    app.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
    self.io_loop.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
    self.asyncio_loop.run_forever()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
    self._run_once()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
    handle._run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
    self._context.run(self._callback, *self._args)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
    return await f(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
    await self.dispatch_shell(msg, subshell_id=subshell_id)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
    await result
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
    await super().execute_request(stream, ident, parent)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
    reply_content = await reply_content
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
    res = shell.run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
    return super().run_cell(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
    result = self._run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
    result = runner(coro)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
    coro.send(None)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
    if await self.run_code(code, result, async_=asy):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "/tmp/ipykernel_503/2124425154.py", line 8, in <module>
    fn(torch.randn(3))
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 992, in compile_wrapper
    return fn(*args, **kwargs)
Step 1: torchdynamo start tracing torch_dynamo_resume_in_fn_at_5 /tmp/ipykernel_503/2124425154.py:5
create_env
TRACE starts_line /tmp/ipykernel_503/2124425154.py:5 in torch_dynamo_resume_in_fn_at_5
        torch._dynamo.graph_break()
TRACE LOAD_CONST True []
TRACE STORE_FAST __is_tracing_resume_prologue [ConstantVariable(bool: True)]
TRACE LOAD_FAST ___stack0 []
TRACE LOAD_CONST False [LazyVariableTracker(unrealized: <class 'NoneType'>)]
TRACE STORE_FAST __is_tracing_resume_prologue [LazyVariableTracker(unrealized: <class 'NoneType'>), ConstantVariable(bool: False)]
TRACE JUMP_ABSOLUTE 66 [LazyVariableTracker(unrealized: <class 'NoneType'>)]
TRACE POP_TOP None [LazyVariableTracker(unrealized: <class 'NoneType'>)]
TRACE starts_line /tmp/ipykernel_503/2124425154.py:6 in torch_dynamo_resume_in_fn_at_5
        with CustomCtxManager():
TRACE LOAD_GLOBAL CustomCtxManager []
TRACE CALL_FUNCTION 0 [LazyVariableTracker(unrealized: <class 'type'>)]
TRACE SETUP_WITH 98 [GenericContextWrappingVariable(CustomCtxManager)]
INLINING <code object __enter__ at 0x7f7fc7f68a80, file "/tmp/ipykernel_503/4148913404.py", line 2>, inlined according trace_rules.lookup inlined by default
TRACE starts_line /tmp/ipykernel_503/4148913404.py:3 in __enter__ (inline depth: 1)
            pass
TRACE LOAD_CONST None []
TRACE RETURN_VALUE None [ConstantVariable(NoneType: None)]
DONE INLINING <code object __enter__ at 0x7f7fc7f68a80, file "/tmp/ipykernel_503/4148913404.py", line 2>
TRACE POP_TOP None [WithExitFunctionVariable(), ConstantVariable(NoneType: None)]
TRACE starts_line /tmp/ipykernel_503/2124425154.py:7 in torch_dynamo_resume_in_fn_at_5
            return x + 1
TRACE LOAD_FAST x [WithExitFunctionVariable()]
TRACE LOAD_CONST 1 [WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'torch.Tensor'>)]
TRACE BINARY_ADD None [WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'torch.Tensor'>), ConstantVariable(int: 1)]
wrap_to_fake L['x'] (3,) StatefulSymbolicContext(dynamic_sizes=[<DimDynamic.STATIC: 2>], dynamic_strides=[<DimDynamic.INFER_STRIDE: 4>], constraint_sizes=[None], constraint_strides=[None], specialize_on=[[]], view_base_context=None, tensor_source=LocalSource(local_name='x', is_input=True, dynamism=None, is_derefed_cell_contents=False), shape_env_to_source_to_symbol_cache={}) <class 'torch.Tensor'>
create_graph_input L_x_ L['x'] FakeTensor(..., size=(3,)) at debug_level 0 before=False
TRACE starts_line /tmp/ipykernel_503/2124425154.py:6 in torch_dynamo_resume_in_fn_at_5
        with CustomCtxManager():
TRACE POP_BLOCK None [WithExitFunctionVariable(), TensorVariable()]
TRACE ROT_TWO None [WithExitFunctionVariable(), TensorVariable()]
TRACE LOAD_CONST None [TensorVariable(), WithExitFunctionVariable()]
TRACE DUP_TOP None [TensorVariable(), WithExitFunctionVariable(), ConstantVariable(NoneType: None)]
TRACE DUP_TOP None [TensorVariable(), WithExitFunctionVariable(), ConstantVariable(NoneType: None), ConstantVariable(NoneType: None)]
TRACE CALL_FUNCTION 3 [TensorVariable(), WithExitFunctionVariable(), ConstantVariable(NoneType: None), ConstantVariable(NoneType: None), ConstantVariable(NoneType: None)]
INLINING <code object __exit__ at 0x7f7f5e1aae40, file "/tmp/ipykernel_503/4148913404.py", line 4>, inlined according trace_rules.lookup inlined by default
TRACE starts_line /tmp/ipykernel_503/4148913404.py:5 in __exit__ (inline depth: 1)
            pass
TRACE LOAD_CONST None []
TRACE RETURN_VALUE None [ConstantVariable(NoneType: None)]
DONE INLINING <code object __exit__ at 0x7f7f5e1aae40, file "/tmp/ipykernel_503/4148913404.py", line 4>
TRACE POP_TOP None [TensorVariable(), ConstantVariable(NoneType: None)]
TRACE RETURN_VALUE None [TensorVariable()]
Step 1: torchdynamo done tracing torch_dynamo_resume_in_fn_at_5 (RETURN_VALUE)
return triggered compile
COMPILING GRAPH due to GraphCompileReason(reason='return_value', user_stack=[<FrameSummary file /tmp/ipykernel_503/2124425154.py, line 6 in torch_dynamo_resume_in_fn_at_5>], graph_break=False)
TRACED GRAPH
 ===== __compiled_fn_23_a7d1065f_19dd_4ceb_8a6e_2aec5c61ef31 =====
 /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/fx/_lazy_graph_module.py class GraphModule(torch.nn.Module):
    def forward(self, L_x_: "f32[3][1]cpu"):
        l_x_ = L_x_

        # File: /tmp/ipykernel_503/2124425154.py:7 in torch_dynamo_resume_in_fn_at_5, code: return x + 1
        add: "f32[3][1]cpu" = l_x_ + 1;  l_x_ = None
        return (add,)
Step 2: calling compiler function eager
Step 2: done compiler function eager
produce_guards
track_symint L['x'].size()[0] 3 None
track_symint L['x'].stride()[0] 1 None
track_symint L['x'].storage_offset() 0 None
Skipping guard L['x'].size()[0] == 3
Skipping guard L['x'].stride()[0] == 1
Skipping guard L['x'].storage_offset() == 0
GUARDS:

TREE_GUARD_MANAGER:
+- RootGuardManager
| +- LAMBDA_GUARD: torch._functorch.aot_autograd.utils.top_saved_tensors_hooks ids == None  # _dynamo/output_graph.py:866 in init_ambient_guards
| +- GLOBAL_STATE: ___check_global_state() against {"allow_bf16_reduce":0,"allow_fp16_reduce":0,"allow_tf32":false,"autocast_state":{"cached_enabled":true,"dtype":[15,5,5,15,5,5,15,15,5,5],"enabled":[false,false,false,false,false,false,false,false,false,false]},"default_dtype":6,"deterministic_algorithms":false,"deterministic_algorithms_warn_only":false,"grad_mode":true,"num_threads":4,"torch_function":true,"torch_function_all_disabled":false}
| +- TORCH_FUNCTION_MODE_STACK: ___check_torch_function_mode_stack()
| +- DEFAULT_DEVICE: utils_device.CURRENT_DEVICE == None                           # _dynamo/output_graph.py:853 in init_ambient_guards
| +- GuardManager: source=L['x'], accessed_by=FrameLocalsGuardAccessor(key='x', framelocals_idx=3), type=<class 'torch.Tensor'>, tag_safe=(True, False)
| | +- TENSOR_MATCH: check_tensor(L['x'], Tensor, DispatchKeySet(CPU, BackendSelect, ADInplaceOrView, AutogradCPU), torch.float32, device=None, requires_grad=False, size=[3], stride=[1])  # return x + 1  # mp/ipykernel_503/2124425154.py:7 in torch_dynamo_resume_in_fn_at_5
| | +- NO_HASATTR: hasattr(L['x'], '_dynamo_dynamic_indices') == False           # return x + 1  # mp/ipykernel_503/2124425154.py:7 in torch_dynamo_resume_in_fn_at_5
| +- GuardManager: source=G, accessed_by=GlobalsGuardAccessor, type=<class 'dict'>, tag_safe=(True, False)
| | +- GuardManager: source=G['CustomCtxManager'], accessed_by=DictGetItemGuardAccessor('CustomCtxManager'), type=<class 'type'>, tag_safe=(True, False)
| | | +- ID_MATCH: ___check_obj_id(G['CustomCtxManager'], 93962434258400), type=<class '__main__.CustomCtxManager'>  # with CustomCtxManager():  # mp/ipykernel_503/2124425154.py:6 in torch_dynamo_resume_in_fn_at_5
Guard eval latency = 44.53 us
put_code_state: no cache key, skipping
run_gc_after_compile: running gc

tensor([1.7313, 2.8806, 1.8631])

There are some context managers where Dynamo can resume after a graph break. Some of these can be found in supported_ctx_manager_classes in torch/_dynamo/variables/torch.py. In general, any context manager represented by a ContextWrappingVariable subclass in torch/_dynamo/variables/ctx_manager.py support resuming after a graph break. For example:

import contextlib
@torch.compile
def fn(x):
    with contextlib.nullcontext():
        with torch.no_grad():
            x = x + 1
            torch._dynamo.graph_break()
            return x + 1
fn(torch.randn(3))

torchdynamo start compiling fn /tmp/ipykernel_503/3152636365.py:2, stack (elided 5 frames):
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
    app.launch_new_instance()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
    app.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
    self.io_loop.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
    self.asyncio_loop.run_forever()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
    self._run_once()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
    handle._run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
    self._context.run(self._callback, *self._args)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
    return await f(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
    await self.dispatch_shell(msg, subshell_id=subshell_id)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
    await result
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
    await super().execute_request(stream, ident, parent)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
    reply_content = await reply_content
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
    res = shell.run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
    return super().run_cell(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
    result = self._run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
    result = runner(coro)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
    coro.send(None)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
    if await self.run_code(code, result, async_=asy):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "/tmp/ipykernel_503/3152636365.py", line 9, in <module>
    fn(torch.randn(3))
Step 1: torchdynamo start tracing fn /tmp/ipykernel_503/3152636365.py:2
create_env
TRACE starts_line /tmp/ipykernel_503/3152636365.py:4 in fn
        with contextlib.nullcontext():
TRACE LOAD_GLOBAL contextlib []
TRACE LOAD_ATTR nullcontext [LazyVariableTracker(unrealized: <class 'module'>)]
TRACE CALL_FUNCTION 0 [LazyVariableTracker(unrealized: <class 'abc.ABCMeta'>)]
TRACE SETUP_WITH 106 [NullContextVariable()]
TRACE POP_TOP None [WithExitFunctionVariable(), ConstantVariable(NoneType: None)]
TRACE starts_line /tmp/ipykernel_503/3152636365.py:5 in fn
            with torch.no_grad():
TRACE LOAD_GLOBAL torch [WithExitFunctionVariable()]
TRACE LOAD_ATTR no_grad [WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'module'>)]
TRACE CALL_FUNCTION 0 [WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'type'>)]
TRACE SETUP_WITH 74 [WithExitFunctionVariable(), GradModeVariable()]
TRACE POP_TOP None [WithExitFunctionVariable(), WithExitFunctionVariable(), ConstantVariable(NoneType: None)]
TRACE starts_line /tmp/ipykernel_503/3152636365.py:6 in fn
                x = x + 1
TRACE LOAD_FAST x [WithExitFunctionVariable(), WithExitFunctionVariable()]
TRACE LOAD_CONST 1 [WithExitFunctionVariable(), WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'torch.Tensor'>)]
TRACE BINARY_ADD None [WithExitFunctionVariable(), WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'torch.Tensor'>), ConstantVariable(int: 1)]
wrap_to_fake L['x'] (3,) StatefulSymbolicContext(dynamic_sizes=[<DimDynamic.STATIC: 2>], dynamic_strides=[<DimDynamic.INFER_STRIDE: 4>], constraint_sizes=[None], constraint_strides=[None], specialize_on=[[]], view_base_context=None, tensor_source=LocalSource(local_name='x', is_input=True, dynamism=None, is_derefed_cell_contents=False), shape_env_to_source_to_symbol_cache={}) <class 'torch.Tensor'>
create_graph_input L_x_ L['x'] FakeTensor(..., size=(3,)) at debug_level 0 before=False
TRACE STORE_FAST x [WithExitFunctionVariable(), WithExitFunctionVariable(), TensorVariable()]
TRACE starts_line /tmp/ipykernel_503/3152636365.py:7 in fn
                torch._dynamo.graph_break()
TRACE LOAD_GLOBAL torch [WithExitFunctionVariable(), WithExitFunctionVariable()]
TRACE LOAD_ATTR _dynamo [WithExitFunctionVariable(), WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'module'>)]
TRACE LOAD_ATTR graph_break [WithExitFunctionVariable(), WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'module'>)]
TRACE CALL_FUNCTION 0 [WithExitFunctionVariable(), WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'function'>)]
Graph break in user code at /tmp/ipykernel_503/3152636365.py:7
Graph Break Reason: Encountered graph break when attempting to trace CALL_FUNCTION: a call to a regular function, e.g. f(x, y):

Call to `torch._dynamo.graph_break()`
  Explanation: User-inserted graph break. Message: None
  Hint: Remove the `torch._dynamo.graph_break()` call.

  Developer debug context: Called `torch._dynamo.graph_break()` with args `[]`, kwargs `{}`

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb0025.html

User code traceback:
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
    app.launch_new_instance()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
    app.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
    self.io_loop.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
    self.asyncio_loop.run_forever()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
    self._run_once()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
    handle._run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
    self._context.run(self._callback, *self._args)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
    return await f(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
    await self.dispatch_shell(msg, subshell_id=subshell_id)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
    await result
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
    await super().execute_request(stream, ident, parent)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
    reply_content = await reply_content
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
    res = shell.run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
    return super().run_cell(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
    result = self._run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
    result = runner(coro)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
    coro.send(None)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
    if await self.run_code(code, result, async_=asy):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "/tmp/ipykernel_503/3152636365.py", line 9, in <module>
    fn(torch.randn(3))
  File "/tmp/ipykernel_503/3152636365.py", line 7, in fn
    torch._dynamo.graph_break()
Restarting analysis due to _dynamo/symbolic_convert.py:252 in fail_and_restart_analysis
Step 1: torchdynamo start tracing fn /tmp/ipykernel_503/3152636365.py:2
create_env
TRACE starts_line /tmp/ipykernel_503/3152636365.py:4 in fn
        with contextlib.nullcontext():
TRACE LOAD_GLOBAL contextlib []
TRACE LOAD_ATTR nullcontext [LazyVariableTracker(unrealized: <class 'module'>)]
TRACE CALL_FUNCTION 0 [LazyVariableTracker(unrealized: <class 'abc.ABCMeta'>)]
TRACE SETUP_WITH 106 [NullContextVariable()]
TRACE POP_TOP None [WithExitFunctionVariable(), ConstantVariable(NoneType: None)]
TRACE starts_line /tmp/ipykernel_503/3152636365.py:5 in fn
            with torch.no_grad():
TRACE LOAD_GLOBAL torch [WithExitFunctionVariable()]
TRACE LOAD_ATTR no_grad [WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'module'>)]
TRACE CALL_FUNCTION 0 [WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'type'>)]
TRACE SETUP_WITH 74 [WithExitFunctionVariable(), GradModeVariable()]
TRACE POP_TOP None [WithExitFunctionVariable(), WithExitFunctionVariable(), ConstantVariable(NoneType: None)]
TRACE starts_line /tmp/ipykernel_503/3152636365.py:6 in fn
                x = x + 1
TRACE LOAD_FAST x [WithExitFunctionVariable(), WithExitFunctionVariable()]
TRACE LOAD_CONST 1 [WithExitFunctionVariable(), WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'torch.Tensor'>)]
TRACE BINARY_ADD None [WithExitFunctionVariable(), WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'torch.Tensor'>), ConstantVariable(int: 1)]
wrap_to_fake L['x'] (3,) StatefulSymbolicContext(dynamic_sizes=[<DimDynamic.STATIC: 2>], dynamic_strides=[<DimDynamic.INFER_STRIDE: 4>], constraint_sizes=[None], constraint_strides=[None], specialize_on=[[]], view_base_context=None, tensor_source=LocalSource(local_name='x', is_input=True, dynamism=None, is_derefed_cell_contents=False), shape_env_to_source_to_symbol_cache={}) <class 'torch.Tensor'>
create_graph_input L_x_ L['x'] FakeTensor(..., size=(3,)) at debug_level 0 before=False
TRACE STORE_FAST x [WithExitFunctionVariable(), WithExitFunctionVariable(), TensorVariable()]
TRACE starts_line /tmp/ipykernel_503/3152636365.py:7 in fn
                torch._dynamo.graph_break()
TRACE LOAD_GLOBAL torch [WithExitFunctionVariable(), WithExitFunctionVariable()]
TRACE LOAD_ATTR _dynamo [WithExitFunctionVariable(), WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'module'>)]
TRACE LOAD_ATTR graph_break [WithExitFunctionVariable(), WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'module'>)]
TRACE CALL_FUNCTION 0 [WithExitFunctionVariable(), WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'function'>)]
CALL_FUNCTION triggered compile
COMPILING GRAPH due to GraphCompileReason(reason='Call to `torch._dynamo.graph_break()`\n  Explanation: User-inserted graph break. Message: None\n  Hint: Remove the `torch._dynamo.graph_break()` call.\n\n  Developer debug context: Called `torch._dynamo.graph_break()` with args `[]`, kwargs `{}`\n\n For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb0025.html', user_stack=[<FrameSummary file /tmp/ipykernel_503/3152636365.py, line 7 in fn>], graph_break=True)
TRACED GRAPH
 ===== __compiled_fn_26_279af000_4486_4a0b_9d2f_12f1b4d39dd8 =====
 /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/fx/_lazy_graph_module.py class GraphModule(torch.nn.Module):
    def forward(self, L_x_: "f32[3][1]cpu"):
        l_x_ = L_x_

        # No stacktrace found for following nodes
        _set_grad_enabled = torch._C._set_grad_enabled(False);  _set_grad_enabled = None

        # File: /tmp/ipykernel_503/3152636365.py:6 in fn, code: x = x + 1
        x: "f32[3][1]cpu" = l_x_ + 1;  l_x_ = None

        # No stacktrace found for following nodes
        _set_grad_enabled_1 = torch._C._set_grad_enabled(True);  _set_grad_enabled_1 = None
        return (x,)
Step 2: calling compiler function eager
Step 2: done compiler function eager
produce_guards
track_symint L['x'].size()[0] 3 None
track_symint L['x'].stride()[0] 1 None
track_symint L['x'].storage_offset() 0 None
Skipping guard L['x'].size()[0] == 3
Skipping guard L['x'].stride()[0] == 1
Skipping guard L['x'].storage_offset() == 0
GUARDS:

TREE_GUARD_MANAGER:
+- RootGuardManager
| +- LAMBDA_GUARD: torch._functorch.aot_autograd.utils.top_saved_tensors_hooks ids == None  # _dynamo/output_graph.py:866 in init_ambient_guards
| +- GLOBAL_STATE: ___check_global_state() against {"allow_bf16_reduce":0,"allow_fp16_reduce":0,"allow_tf32":false,"autocast_state":{"cached_enabled":true,"dtype":[15,5,5,15,5,5,15,15,5,5],"enabled":[false,false,false,false,false,false,false,false,false,false]},"default_dtype":6,"deterministic_algorithms":false,"deterministic_algorithms_warn_only":false,"grad_mode":true,"num_threads":4,"torch_function":true,"torch_function_all_disabled":false}
| +- TORCH_FUNCTION_MODE_STACK: ___check_torch_function_mode_stack()
| +- DEFAULT_DEVICE: utils_device.CURRENT_DEVICE == None                           # _dynamo/output_graph.py:853 in init_ambient_guards
| +- GuardManager: source=L['x'], accessed_by=FrameLocalsGuardAccessor(key='x', framelocals_idx=0), type=<class 'torch.Tensor'>, tag_safe=(True, False)
| | +- TENSOR_MATCH: check_tensor(L['x'], Tensor, DispatchKeySet(CPU, BackendSelect, ADInplaceOrView, AutogradCPU), torch.float32, device=None, requires_grad=False, size=[3], stride=[1])  # x = x + 1  # mp/ipykernel_503/3152636365.py:6 in fn
| | +- NO_HASATTR: hasattr(L['x'], '_dynamo_dynamic_indices') == False           # x = x + 1  # mp/ipykernel_503/3152636365.py:6 in fn
| +- GuardManager: source=G, accessed_by=GlobalsGuardAccessor, type=<class 'dict'>, tag_safe=(False, False)
| | +- GuardManager: source=G['torch'], accessed_by=DictGetItemGuardAccessor('torch'), type=<class 'module'>, tag_safe=(False, False)
| | | +- ID_MATCH: ___check_obj_id(G['torch'], 140186730360128), type=<module 'torch' from '/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/__init__.py'>  # with torch.no_grad():  # mp/ipykernel_503/3152636365.py:5 in fn
| | | +- GuardManager: source=G['torch']._dynamo, accessed_by=GetAttrGuardAccessor(_dynamo), type=<class 'module'>, tag_safe=(False, False)
| | | | +- ID_MATCH: ___check_obj_id(G['torch']._dynamo, 140185246904016), type=<module 'torch._dynamo' from '/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/__init__.py'>  # torch._dynamo.graph_break()  # mp/ipykernel_503/3152636365.py:7 in fn
| | | | +- GuardManager: source=G['torch']._dynamo.graph_break, accessed_by=GetAttrGuardAccessor(graph_break), type=<class 'function'>, tag_safe=(True, False)
| | | | | +- GuardManager: source=G['torch']._dynamo.graph_break.__code__, accessed_by=CodeGuardAccessor, type=<class 'code'>, tag_safe=(True, False)
| | | | | | +- ID_MATCH: ___check_obj_id(G['torch']._dynamo.graph_break.__code__, 140185053639792), type=<code object graph_break at 0x7f7f60533470, file "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/decorators.py", line 290>  # torch._dynamo.graph_break()  # mp/ipykernel_503/3152636365.py:7 in fn
| | | +- GuardManager: source=G['torch'].no_grad, accessed_by=GetAttrGuardAccessor(no_grad), type=<class 'type'>, tag_safe=(True, False)
| | | | +- ID_MATCH: ___check_obj_id(G['torch'].no_grad, 93962395455936), type=<class 'torch.autograd.grad_mode.no_grad'>  # with torch.no_grad():  # mp/ipykernel_503/3152636365.py:5 in fn
| | +- GuardManager: source=G['contextlib'], accessed_by=DictGetItemGuardAccessor('contextlib'), type=<class 'module'>, tag_safe=(False, False)
| | | +- ID_MATCH: ___check_obj_id(G['contextlib'], 140186801825216), type=<module 'contextlib' from '/opt/conda/envs/py_3.10/lib/python3.10/contextlib.py'>  # with contextlib.nullcontext():  # mp/ipykernel_503/3152636365.py:4 in fn
| | | +- GuardManager: source=G['contextlib'].nullcontext, accessed_by=GetAttrGuardAccessor(nullcontext), type=<class 'abc.ABCMeta'>, tag_safe=(True, False)
| | | | +- ID_MATCH: ___check_obj_id(G['contextlib'].nullcontext, 93962319450480), type=<class 'contextlib.nullcontext'>  # with contextlib.nullcontext():  # mp/ipykernel_503/3152636365.py:4 in fn
Guard eval latency = 451.13 us
put_code_state: no cache key, skipping
run_gc_after_compile: running gc
skipping because no torch.* __init__             /opt/conda/envs/py_3.10/lib/python3.10/contextlib.py 732
skipping because no torch.* __enter__             /opt/conda/envs/py_3.10/lib/python3.10/contextlib.py 735
skipping: __init__ (reason: in skipfiles, file: /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/autograd/grad_mode.py)
skipping: __enter__ (reason: in skipfiles, file: /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/autograd/grad_mode.py)
skipping: __exit__ (reason: in skipfiles, file: /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/autograd/grad_mode.py)
skipping because no torch.* __exit__             /opt/conda/envs/py_3.10/lib/python3.10/contextlib.py 738
torchdynamo start compiling torch_dynamo_resume_in_fn_at_7 /tmp/ipykernel_503/3152636365.py:7, stack (elided 5 frames):
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
    app.launch_new_instance()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
    app.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
    self.io_loop.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
    self.asyncio_loop.run_forever()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
    self._run_once()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
    handle._run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
    self._context.run(self._callback, *self._args)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
    return await f(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
    await self.dispatch_shell(msg, subshell_id=subshell_id)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
    await result
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
    await super().execute_request(stream, ident, parent)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
    reply_content = await reply_content
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
    res = shell.run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
    return super().run_cell(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
    result = self._run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
    result = runner(coro)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
    coro.send(None)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
    if await self.run_code(code, result, async_=asy):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "/tmp/ipykernel_503/3152636365.py", line 9, in <module>
    fn(torch.randn(3))
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 992, in compile_wrapper
    return fn(*args, **kwargs)
Step 1: torchdynamo start tracing torch_dynamo_resume_in_fn_at_7 /tmp/ipykernel_503/3152636365.py:7
create_env
TRACE starts_line /tmp/ipykernel_503/3152636365.py:7 in torch_dynamo_resume_in_fn_at_7
                torch._dynamo.graph_break()
TRACE LOAD_CONST True []
TRACE STORE_FAST __is_tracing_resume_prologue [ConstantVariable(bool: True)]
TRACE LOAD_FAST ___stack0 []
TRACE CALL_FUNCTION 0 [LazyVariableTracker(unrealized: <class 'abc.ABCMeta'>)]
TRACE NOP None [NullContextVariable()]
TRACE SETUP_WITH 96 [NullContextVariable()]
TRACE POP_TOP None [WithExitFunctionVariable(), ConstantVariable(NoneType: None)]
TRACE NOP None [WithExitFunctionVariable()]
TRACE LOAD_FAST ___stack1 [WithExitFunctionVariable()]
TRACE LOAD_CONST False [WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'type'>)]
TRACE CALL_FUNCTION 1 [WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'type'>), ConstantVariable(bool: False)]
TRACE NOP None [WithExitFunctionVariable(), GradModeVariable()]
TRACE SETUP_WITH 56 [WithExitFunctionVariable(), GradModeVariable()]
TRACE POP_TOP None [WithExitFunctionVariable(), WithExitFunctionVariable(), ConstantVariable(NoneType: None)]
TRACE NOP None [WithExitFunctionVariable(), WithExitFunctionVariable()]
TRACE LOAD_FAST ___stack2 [WithExitFunctionVariable(), WithExitFunctionVariable()]
TRACE LOAD_CONST False [WithExitFunctionVariable(), WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'NoneType'>)]
TRACE STORE_FAST __is_tracing_resume_prologue [WithExitFunctionVariable(), WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'NoneType'>), ConstantVariable(bool: False)]
TRACE JUMP_ABSOLUTE 158 [WithExitFunctionVariable(), WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'NoneType'>)]
TRACE POP_TOP None [WithExitFunctionVariable(), WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'NoneType'>)]
TRACE starts_line /tmp/ipykernel_503/3152636365.py:8 in torch_dynamo_resume_in_fn_at_7
                return x + 1
TRACE LOAD_FAST x [WithExitFunctionVariable(), WithExitFunctionVariable()]
TRACE LOAD_CONST 1 [WithExitFunctionVariable(), WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'torch.Tensor'>)]
TRACE BINARY_ADD None [WithExitFunctionVariable(), WithExitFunctionVariable(), LazyVariableTracker(unrealized: <class 'torch.Tensor'>), ConstantVariable(int: 1)]
wrap_to_fake L['x'] (3,) StatefulSymbolicContext(dynamic_sizes=[<DimDynamic.STATIC: 2>], dynamic_strides=[<DimDynamic.INFER_STRIDE: 4>], constraint_sizes=[None], constraint_strides=[None], specialize_on=[[]], view_base_context=None, tensor_source=LocalSource(local_name='x', is_input=True, dynamism=None, is_derefed_cell_contents=False), shape_env_to_source_to_symbol_cache={}) <class 'torch.Tensor'>
create_graph_input L_x_ L['x'] FakeTensor(..., size=(3,)) at debug_level 0 before=False
TRACE starts_line /tmp/ipykernel_503/3152636365.py:5 in torch_dynamo_resume_in_fn_at_7
            with torch.no_grad():
TRACE POP_BLOCK None [WithExitFunctionVariable(), WithExitFunctionVariable(), TensorVariable()]
TRACE ROT_TWO None [WithExitFunctionVariable(), WithExitFunctionVariable(), TensorVariable()]
TRACE LOAD_CONST None [WithExitFunctionVariable(), TensorVariable(), WithExitFunctionVariable()]
TRACE DUP_TOP None [WithExitFunctionVariable(), TensorVariable(), WithExitFunctionVariable(), ConstantVariable(NoneType: None)]
TRACE DUP_TOP None [WithExitFunctionVariable(), TensorVariable(), WithExitFunctionVariable(), ConstantVariable(NoneType: None), ConstantVariable(NoneType: None)]
TRACE CALL_FUNCTION 3 [WithExitFunctionVariable(), TensorVariable(), WithExitFunctionVariable(), ConstantVariable(NoneType: None), ConstantVariable(NoneType: None), ConstantVariable(NoneType: None)]
TRACE POP_TOP None [WithExitFunctionVariable(), TensorVariable(), ConstantVariable(NoneType: None)]
TRACE starts_line /tmp/ipykernel_503/3152636365.py:4 in torch_dynamo_resume_in_fn_at_7
        with contextlib.nullcontext():
TRACE POP_BLOCK None [WithExitFunctionVariable(), TensorVariable()]
TRACE ROT_TWO None [WithExitFunctionVariable(), TensorVariable()]
TRACE LOAD_CONST None [TensorVariable(), WithExitFunctionVariable()]
TRACE DUP_TOP None [TensorVariable(), WithExitFunctionVariable(), ConstantVariable(NoneType: None)]
TRACE DUP_TOP None [TensorVariable(), WithExitFunctionVariable(), ConstantVariable(NoneType: None), ConstantVariable(NoneType: None)]
TRACE CALL_FUNCTION 3 [TensorVariable(), WithExitFunctionVariable(), ConstantVariable(NoneType: None), ConstantVariable(NoneType: None), ConstantVariable(NoneType: None)]
TRACE POP_TOP None [TensorVariable(), ConstantVariable(NoneType: None)]
TRACE RETURN_VALUE None [TensorVariable()]
Step 1: torchdynamo done tracing torch_dynamo_resume_in_fn_at_7 (RETURN_VALUE)
return triggered compile
COMPILING GRAPH due to GraphCompileReason(reason='return_value', user_stack=[<FrameSummary file /tmp/ipykernel_503/3152636365.py, line 4 in torch_dynamo_resume_in_fn_at_7>], graph_break=False)
TRACED GRAPH
 ===== __compiled_fn_31_aad698c6_7e99_4668_9290_26b6e89d889a =====
 /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/fx/_lazy_graph_module.py class GraphModule(torch.nn.Module):
    def forward(self, L_x_: "f32[3][1]cpu"):
        l_x_ = L_x_

        # No stacktrace found for following nodes
        _set_grad_enabled = torch._C._set_grad_enabled(False);  _set_grad_enabled = None

        # File: /tmp/ipykernel_503/3152636365.py:8 in torch_dynamo_resume_in_fn_at_7, code: return x + 1
        add: "f32[3][1]cpu" = l_x_ + 1;  l_x_ = None

        # No stacktrace found for following nodes
        _set_grad_enabled_1 = torch._C._set_grad_enabled(True);  _set_grad_enabled_1 = None
        return (add,)
Step 2: calling compiler function eager
Step 2: done compiler function eager
produce_guards
track_symint L['x'].size()[0] 3 None
track_symint L['x'].stride()[0] 1 None
track_symint L['x'].storage_offset() 0 None
Skipping guard L['x'].size()[0] == 3
Skipping guard L['x'].stride()[0] == 1
Skipping guard L['x'].storage_offset() == 0
GUARDS:

TREE_GUARD_MANAGER:
+- RootGuardManager
| +- LAMBDA_GUARD: torch._functorch.aot_autograd.utils.top_saved_tensors_hooks ids == None  # _dynamo/output_graph.py:866 in init_ambient_guards
| +- GLOBAL_STATE: ___check_global_state() against {"allow_bf16_reduce":0,"allow_fp16_reduce":0,"allow_tf32":false,"autocast_state":{"cached_enabled":true,"dtype":[15,5,5,15,5,5,15,15,5,5],"enabled":[false,false,false,false,false,false,false,false,false,false]},"default_dtype":6,"deterministic_algorithms":false,"deterministic_algorithms_warn_only":false,"grad_mode":true,"num_threads":4,"torch_function":true,"torch_function_all_disabled":false}
| +- TORCH_FUNCTION_MODE_STACK: ___check_torch_function_mode_stack()
| +- DEFAULT_DEVICE: utils_device.CURRENT_DEVICE == None                           # _dynamo/output_graph.py:853 in init_ambient_guards
| +- GuardManager: source=L['x'], accessed_by=FrameLocalsGuardAccessor(key='x', framelocals_idx=5), type=<class 'torch.Tensor'>, tag_safe=(True, False)
| | +- TENSOR_MATCH: check_tensor(L['x'], Tensor, DispatchKeySet(CPU, BackendSelect, ADInplaceOrView, AutogradCPU), torch.float32, device=None, requires_grad=False, size=[3], stride=[1])  # return x + 1  # mp/ipykernel_503/3152636365.py:8 in torch_dynamo_resume_in_fn_at_7
| | +- NO_HASATTR: hasattr(L['x'], '_dynamo_dynamic_indices') == False           # return x + 1  # mp/ipykernel_503/3152636365.py:8 in torch_dynamo_resume_in_fn_at_7
| +- GuardManager: source=L['___stack0'], accessed_by=FrameLocalsGuardAccessor(key='___stack0', framelocals_idx=2), type=<class 'abc.ABCMeta'>, tag_safe=(True, False)
| | +- ID_MATCH: ___check_obj_id(L['___stack0'], 93962319450480), type=<class 'contextlib.nullcontext'>  # torch._dynamo.graph_break()  # mp/ipykernel_503/3152636365.py:7 in torch_dynamo_resume_in_fn_at_7
| +- GuardManager: source=L['___stack1'], accessed_by=FrameLocalsGuardAccessor(key='___stack1', framelocals_idx=3), type=<class 'type'>, tag_safe=(True, False)
| | +- ID_MATCH: ___check_obj_id(L['___stack1'], 93962395461584), type=<class 'torch.autograd.grad_mode.set_grad_enabled'>  # torch._dynamo.graph_break()  # mp/ipykernel_503/3152636365.py:7 in torch_dynamo_resume_in_fn_at_7
Guard eval latency = 340.18 us
put_code_state: no cache key, skipping
run_gc_after_compile: running gc

tensor([2.6017, 3.3658, 2.0726])

Graph Break in a Try Block

A graph break in a try block cannot be resumed:

@torch.compile
def fn(x):
    try:
        x = x + 1
        torch._dynamo.graph_break()
        return x + 1
    except Exception as e:
        pass
fn(torch.randn(3))

torchdynamo start compiling fn /tmp/ipykernel_503/2546874632.py:1, stack (elided 5 frames):
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
    app.launch_new_instance()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
    app.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
    self.io_loop.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
    self.asyncio_loop.run_forever()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
    self._run_once()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
    handle._run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
    self._context.run(self._callback, *self._args)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
    return await f(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
    await self.dispatch_shell(msg, subshell_id=subshell_id)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
    await result
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
    await super().execute_request(stream, ident, parent)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
    reply_content = await reply_content
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
    res = shell.run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
    return super().run_cell(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
    result = self._run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
    result = runner(coro)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
    coro.send(None)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
    if await self.run_code(code, result, async_=asy):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "/tmp/ipykernel_503/2546874632.py", line 9, in <module>
    fn(torch.randn(3))
Step 1: torchdynamo start tracing fn /tmp/ipykernel_503/2546874632.py:1
create_env
TRACE starts_line /tmp/ipykernel_503/2546874632.py:3 in fn
        try:
TRACE SETUP_FINALLY 30 []
TRACE starts_line /tmp/ipykernel_503/2546874632.py:4 in fn
            x = x + 1
TRACE LOAD_FAST x []
TRACE LOAD_CONST 1 [LazyVariableTracker(unrealized: <class 'torch.Tensor'>)]
TRACE BINARY_ADD None [LazyVariableTracker(unrealized: <class 'torch.Tensor'>), ConstantVariable(int: 1)]
wrap_to_fake L['x'] (3,) StatefulSymbolicContext(dynamic_sizes=[<DimDynamic.STATIC: 2>], dynamic_strides=[<DimDynamic.INFER_STRIDE: 4>], constraint_sizes=[None], constraint_strides=[None], specialize_on=[[]], view_base_context=None, tensor_source=LocalSource(local_name='x', is_input=True, dynamism=None, is_derefed_cell_contents=False), shape_env_to_source_to_symbol_cache={}) <class 'torch.Tensor'>
create_graph_input L_x_ L['x'] FakeTensor(..., size=(3,)) at debug_level 0 before=False
TRACE STORE_FAST x [TensorVariable()]
TRACE starts_line /tmp/ipykernel_503/2546874632.py:5 in fn
            torch._dynamo.graph_break()
TRACE LOAD_GLOBAL torch []
TRACE LOAD_ATTR _dynamo [LazyVariableTracker(unrealized: <class 'module'>)]
TRACE LOAD_ATTR graph_break [LazyVariableTracker(unrealized: <class 'module'>)]
TRACE CALL_FUNCTION 0 [LazyVariableTracker(unrealized: <class 'function'>)]
empty checkpoint - cannot resume from graph break
Graph break in user code at /tmp/ipykernel_503/2546874632.py:5
Graph Break Reason: Failed to handle graph break gracefully. Skipping the function and falling back to eager. Graph break encountered:

Call to `torch._dynamo.graph_break()`
  Explanation: User-inserted graph break. Message: None
  Hint: Remove the `torch._dynamo.graph_break()` call.

  Developer debug context: Called `torch._dynamo.graph_break()` with args `[]`, kwargs `{}`

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb0025.html

User code traceback:
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
    app.launch_new_instance()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
    app.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
    self.io_loop.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
    self.asyncio_loop.run_forever()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
    self._run_once()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
    handle._run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
    self._context.run(self._callback, *self._args)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
    return await f(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
    await self.dispatch_shell(msg, subshell_id=subshell_id)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
    await result
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
    await super().execute_request(stream, ident, parent)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
    reply_content = await reply_content
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
    res = shell.run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
    return super().run_cell(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
    result = self._run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
    result = runner(coro)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
    coro.send(None)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
    if await self.run_code(code, result, async_=asy):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "/tmp/ipykernel_503/2546874632.py", line 9, in <module>
    fn(torch.randn(3))
  File "/tmp/ipykernel_503/2546874632.py", line 5, in fn
    torch._dynamo.graph_break()
run_gc_after_compile: running gc
WON'T CONVERT fn /tmp/ipykernel_503/2546874632.py line 1 
due to: 
Traceback (most recent call last):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1975, in __call__
    result = self._inner_convert(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 715, in __call__
    result = _compile(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1772, in _compile
    guarded_code, tracer_output = compile_inner(code, one_graph, hooks)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_utils_internal.py", line 96, in wrapper_function
    return function(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1445, in compile_inner
    return _compile_inner(code, one_graph, hooks)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1479, in _compile_inner
    dynamo_output = compile_frame(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1353, in compile_frame
    bytecode, tracer_output = transform_code_object(code, transform)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/bytecode_transformation.py", line 1608, in transform_code_object
    tracer_output = transformations(instructions, code_options)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1325, in transform
    tracer_output = trace_frame(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 327, in _fn
    return fn(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 849, in trace_frame
    run_tracer()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 830, in run_tracer
    tracer.run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1675, in run
    while self.step():
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1340, in step
    self.dispatch_table[inst.opcode](self, inst)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 888, in wrapper
    return inner_fn(self, inst)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 2605, in CALL_FUNCTION
    self.call_function(fn, args, {})
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1247, in call_function
    self.push(fn.call_function(self, args, kwargs))  # type: ignore[arg-type]
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/variables/lazy.py", line 433, in realize_and_forward
    return getattr(self.realize(), name)(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/variables/functions.py", line 1939, in call_function
    unimplemented(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/exc.py", line 634, in unimplemented
    raise Unsupported(msg, gb_type, skip_frame)
torch._dynamo.exc.Unsupported: Call to `torch._dynamo.graph_break()`
  Explanation: User-inserted graph break. Message: None
  Hint: Remove the `torch._dynamo.graph_break()` call.

  Developer debug context: Called `torch._dynamo.graph_break()` with args `[]`, kwargs `{}`

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb0025.html

from user code:
   File "/tmp/ipykernel_503/2546874632.py", line 5, in fn
    torch._dynamo.graph_break()

Set TORCHDYNAMO_VERBOSE=1 for the internal stack trace (please do this especially if you're reporting a bug to PyTorch). For even more developer context, set TORCH_LOGS="+dynamo"

Traceback (most recent call last):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1975, in __call__
    result = self._inner_convert(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 715, in __call__
    result = _compile(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1772, in _compile
    guarded_code, tracer_output = compile_inner(code, one_graph, hooks)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_utils_internal.py", line 96, in wrapper_function
    return function(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1445, in compile_inner
    return _compile_inner(code, one_graph, hooks)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1479, in _compile_inner
    dynamo_output = compile_frame(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1353, in compile_frame
    bytecode, tracer_output = transform_code_object(code, transform)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/bytecode_transformation.py", line 1608, in transform_code_object
    tracer_output = transformations(instructions, code_options)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1325, in transform
    tracer_output = trace_frame(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 327, in _fn
    return fn(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 849, in trace_frame
    run_tracer()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 830, in run_tracer
    tracer.run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1675, in run
    while self.step():
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1340, in step
    self.dispatch_table[inst.opcode](self, inst)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 888, in wrapper
    return inner_fn(self, inst)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 2605, in CALL_FUNCTION
    self.call_function(fn, args, {})
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1247, in call_function
    self.push(fn.call_function(self, args, kwargs))  # type: ignore[arg-type]
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/variables/lazy.py", line 433, in realize_and_forward
    return getattr(self.realize(), name)(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/variables/functions.py", line 1939, in call_function
    unimplemented(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/exc.py", line 634, in unimplemented
    raise Unsupported(msg, gb_type, skip_frame)
torch._dynamo.exc.Unsupported: Call to `torch._dynamo.graph_break()`
  Explanation: User-inserted graph break. Message: None
  Hint: Remove the `torch._dynamo.graph_break()` call.

  Developer debug context: Called `torch._dynamo.graph_break()` with args `[]`, kwargs `{}`

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb0025.html

from user code:
   File "/tmp/ipykernel_503/2546874632.py", line 5, in fn
    torch._dynamo.graph_break()

Set TORCHDYNAMO_VERBOSE=1 for the internal stack trace (please do this especially if you're reporting a bug to PyTorch). For even more developer context, set TORCH_LOGS="+dynamo"

tensor([3.5098, 1.1032, 2.5564])

We can avoid skipping by moving the graph break outside of the try block:

@torch.compile
def fn(x):
    try:
        x = x + 1
    except Exception as e:
        pass
    torch._dynamo.graph_break()
    try:
        return x + 1
    except Exception as e:
        pass
fn(torch.randn(3))

torchdynamo start compiling fn /tmp/ipykernel_503/3015389759.py:1, stack (elided 5 frames):
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
    app.launch_new_instance()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
    app.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
    self.io_loop.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
    self.asyncio_loop.run_forever()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
    self._run_once()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
    handle._run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
    self._context.run(self._callback, *self._args)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
    return await f(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
    await self.dispatch_shell(msg, subshell_id=subshell_id)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
    await result
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
    await super().execute_request(stream, ident, parent)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
    reply_content = await reply_content
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
    res = shell.run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
    return super().run_cell(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
    result = self._run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
    result = runner(coro)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
    coro.send(None)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
    if await self.run_code(code, result, async_=asy):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "/tmp/ipykernel_503/3015389759.py", line 12, in <module>
    fn(torch.randn(3))
Step 1: torchdynamo start tracing fn /tmp/ipykernel_503/3015389759.py:1
create_env
TRACE starts_line /tmp/ipykernel_503/3015389759.py:3 in fn
        try:
TRACE SETUP_FINALLY 14 []
TRACE starts_line /tmp/ipykernel_503/3015389759.py:4 in fn
            x = x + 1
TRACE LOAD_FAST x []
TRACE LOAD_CONST 1 [LazyVariableTracker(unrealized: <class 'torch.Tensor'>)]
TRACE BINARY_ADD None [LazyVariableTracker(unrealized: <class 'torch.Tensor'>), ConstantVariable(int: 1)]
wrap_to_fake L['x'] (3,) StatefulSymbolicContext(dynamic_sizes=[<DimDynamic.STATIC: 2>], dynamic_strides=[<DimDynamic.INFER_STRIDE: 4>], constraint_sizes=[None], constraint_strides=[None], specialize_on=[[]], view_base_context=None, tensor_source=LocalSource(local_name='x', is_input=True, dynamism=None, is_derefed_cell_contents=False), shape_env_to_source_to_symbol_cache={}) <class 'torch.Tensor'>
create_graph_input L_x_ L['x'] FakeTensor(..., size=(3,)) at debug_level 0 before=False
TRACE STORE_FAST x [TensorVariable()]
TRACE POP_BLOCK None []
TRACE JUMP_FORWARD 50 []
TRACE starts_line /tmp/ipykernel_503/3015389759.py:7 in fn
        torch._dynamo.graph_break()
TRACE LOAD_GLOBAL torch []
TRACE LOAD_ATTR _dynamo [LazyVariableTracker(unrealized: <class 'module'>)]
TRACE LOAD_ATTR graph_break [LazyVariableTracker(unrealized: <class 'module'>)]
TRACE CALL_FUNCTION 0 [LazyVariableTracker(unrealized: <class 'function'>)]
Graph break in user code at /tmp/ipykernel_503/3015389759.py:7
Graph Break Reason: Encountered graph break when attempting to trace CALL_FUNCTION: a call to a regular function, e.g. f(x, y):

Call to `torch._dynamo.graph_break()`
  Explanation: User-inserted graph break. Message: None
  Hint: Remove the `torch._dynamo.graph_break()` call.

  Developer debug context: Called `torch._dynamo.graph_break()` with args `[]`, kwargs `{}`

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb0025.html

User code traceback:
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
    app.launch_new_instance()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
    app.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
    self.io_loop.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
    self.asyncio_loop.run_forever()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
    self._run_once()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
    handle._run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
    self._context.run(self._callback, *self._args)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
    return await f(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
    await self.dispatch_shell(msg, subshell_id=subshell_id)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
    await result
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
    await super().execute_request(stream, ident, parent)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
    reply_content = await reply_content
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
    res = shell.run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
    return super().run_cell(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
    result = self._run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
    result = runner(coro)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
    coro.send(None)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
    if await self.run_code(code, result, async_=asy):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "/tmp/ipykernel_503/3015389759.py", line 12, in <module>
    fn(torch.randn(3))
  File "/tmp/ipykernel_503/3015389759.py", line 7, in fn
    torch._dynamo.graph_break()
Restarting analysis due to _dynamo/symbolic_convert.py:252 in fail_and_restart_analysis
Step 1: torchdynamo start tracing fn /tmp/ipykernel_503/3015389759.py:1
create_env
TRACE starts_line /tmp/ipykernel_503/3015389759.py:3 in fn
        try:
TRACE SETUP_FINALLY 14 []
TRACE starts_line /tmp/ipykernel_503/3015389759.py:4 in fn
            x = x + 1
TRACE LOAD_FAST x []
TRACE LOAD_CONST 1 [LazyVariableTracker(unrealized: <class 'torch.Tensor'>)]
TRACE BINARY_ADD None [LazyVariableTracker(unrealized: <class 'torch.Tensor'>), ConstantVariable(int: 1)]
wrap_to_fake L['x'] (3,) StatefulSymbolicContext(dynamic_sizes=[<DimDynamic.STATIC: 2>], dynamic_strides=[<DimDynamic.INFER_STRIDE: 4>], constraint_sizes=[None], constraint_strides=[None], specialize_on=[[]], view_base_context=None, tensor_source=LocalSource(local_name='x', is_input=True, dynamism=None, is_derefed_cell_contents=False), shape_env_to_source_to_symbol_cache={}) <class 'torch.Tensor'>
create_graph_input L_x_ L['x'] FakeTensor(..., size=(3,)) at debug_level 0 before=False
TRACE STORE_FAST x [TensorVariable()]
TRACE POP_BLOCK None []
TRACE JUMP_FORWARD 50 []
TRACE starts_line /tmp/ipykernel_503/3015389759.py:7 in fn
        torch._dynamo.graph_break()
TRACE LOAD_GLOBAL torch []
TRACE LOAD_ATTR _dynamo [LazyVariableTracker(unrealized: <class 'module'>)]
TRACE LOAD_ATTR graph_break [LazyVariableTracker(unrealized: <class 'module'>)]
TRACE CALL_FUNCTION 0 [LazyVariableTracker(unrealized: <class 'function'>)]
CALL_FUNCTION triggered compile
COMPILING GRAPH due to GraphCompileReason(reason='Call to `torch._dynamo.graph_break()`\n  Explanation: User-inserted graph break. Message: None\n  Hint: Remove the `torch._dynamo.graph_break()` call.\n\n  Developer debug context: Called `torch._dynamo.graph_break()` with args `[]`, kwargs `{}`\n\n For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb0025.html', user_stack=[<FrameSummary file /tmp/ipykernel_503/3015389759.py, line 7 in fn>], graph_break=True)
TRACED GRAPH
 ===== __compiled_fn_35_2d9d4b02_39fa_4006_a06f_63c214207f5a =====
 /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/fx/_lazy_graph_module.py class GraphModule(torch.nn.Module):
    def forward(self, L_x_: "f32[3][1]cpu"):
        l_x_ = L_x_

        # File: /tmp/ipykernel_503/3015389759.py:4 in fn, code: x = x + 1
        x: "f32[3][1]cpu" = l_x_ + 1;  l_x_ = None
        return (x,)
Step 2: calling compiler function eager
Step 2: done compiler function eager
produce_guards
track_symint L['x'].size()[0] 3 None
track_symint L['x'].stride()[0] 1 None
track_symint L['x'].storage_offset() 0 None
Skipping guard L['x'].size()[0] == 3
Skipping guard L['x'].stride()[0] == 1
Skipping guard L['x'].storage_offset() == 0
GUARDS:

TREE_GUARD_MANAGER:
+- RootGuardManager
| +- LAMBDA_GUARD: torch._functorch.aot_autograd.utils.top_saved_tensors_hooks ids == None  # _dynamo/output_graph.py:866 in init_ambient_guards
| +- GLOBAL_STATE: ___check_global_state() against {"allow_bf16_reduce":0,"allow_fp16_reduce":0,"allow_tf32":false,"autocast_state":{"cached_enabled":true,"dtype":[15,5,5,15,5,5,15,15,5,5],"enabled":[false,false,false,false,false,false,false,false,false,false]},"default_dtype":6,"deterministic_algorithms":false,"deterministic_algorithms_warn_only":false,"grad_mode":true,"num_threads":4,"torch_function":true,"torch_function_all_disabled":false}
| +- TORCH_FUNCTION_MODE_STACK: ___check_torch_function_mode_stack()
| +- DEFAULT_DEVICE: utils_device.CURRENT_DEVICE == None                           # _dynamo/output_graph.py:853 in init_ambient_guards
| +- GuardManager: source=L['x'], accessed_by=FrameLocalsGuardAccessor(key='x', framelocals_idx=0), type=<class 'torch.Tensor'>, tag_safe=(True, False)
| | +- TENSOR_MATCH: check_tensor(L['x'], Tensor, DispatchKeySet(CPU, BackendSelect, ADInplaceOrView, AutogradCPU), torch.float32, device=None, requires_grad=False, size=[3], stride=[1])  # x = x + 1  # mp/ipykernel_503/3015389759.py:4 in fn
| | +- NO_HASATTR: hasattr(L['x'], '_dynamo_dynamic_indices') == False           # x = x + 1  # mp/ipykernel_503/3015389759.py:4 in fn
| +- GuardManager: source=G, accessed_by=GlobalsGuardAccessor, type=<class 'dict'>, tag_safe=(False, False)
| | +- GuardManager: source=G['torch'], accessed_by=DictGetItemGuardAccessor('torch'), type=<class 'module'>, tag_safe=(False, False)
| | | +- ID_MATCH: ___check_obj_id(G['torch'], 140186730360128), type=<module 'torch' from '/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/__init__.py'>  # torch._dynamo.graph_break()  # mp/ipykernel_503/3015389759.py:7 in fn
| | | +- GuardManager: source=G['torch']._dynamo, accessed_by=GetAttrGuardAccessor(_dynamo), type=<class 'module'>, tag_safe=(False, False)
| | | | +- ID_MATCH: ___check_obj_id(G['torch']._dynamo, 140185246904016), type=<module 'torch._dynamo' from '/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/__init__.py'>  # torch._dynamo.graph_break()  # mp/ipykernel_503/3015389759.py:7 in fn
| | | | +- GuardManager: source=G['torch']._dynamo.graph_break, accessed_by=GetAttrGuardAccessor(graph_break), type=<class 'function'>, tag_safe=(True, False)
| | | | | +- GuardManager: source=G['torch']._dynamo.graph_break.__code__, accessed_by=CodeGuardAccessor, type=<class 'code'>, tag_safe=(True, False)
| | | | | | +- ID_MATCH: ___check_obj_id(G['torch']._dynamo.graph_break.__code__, 140185053639792), type=<code object graph_break at 0x7f7f60533470, file "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/decorators.py", line 290>  # torch._dynamo.graph_break()  # mp/ipykernel_503/3015389759.py:7 in fn
Guard eval latency = 282.49 us
put_code_state: no cache key, skipping
run_gc_after_compile: running gc
torchdynamo start compiling torch_dynamo_resume_in_fn_at_7 /tmp/ipykernel_503/3015389759.py:7, stack (elided 5 frames):
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
    app.launch_new_instance()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
    app.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
    self.io_loop.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
    self.asyncio_loop.run_forever()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
    self._run_once()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
    handle._run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
    self._context.run(self._callback, *self._args)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
    return await f(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
    await self.dispatch_shell(msg, subshell_id=subshell_id)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
    await result
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
    await super().execute_request(stream, ident, parent)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
    reply_content = await reply_content
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
    res = shell.run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
    return super().run_cell(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
    result = self._run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
    result = runner(coro)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
    coro.send(None)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
    if await self.run_code(code, result, async_=asy):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "/tmp/ipykernel_503/3015389759.py", line 12, in <module>
    fn(torch.randn(3))
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 992, in compile_wrapper
    return fn(*args, **kwargs)
Step 1: torchdynamo start tracing torch_dynamo_resume_in_fn_at_7 /tmp/ipykernel_503/3015389759.py:7
create_env
TRACE starts_line /tmp/ipykernel_503/3015389759.py:7 in torch_dynamo_resume_in_fn_at_7
        torch._dynamo.graph_break()
TRACE LOAD_CONST True []
TRACE STORE_FAST __is_tracing_resume_prologue [ConstantVariable(bool: True)]
TRACE LOAD_FAST ___stack0 []
TRACE LOAD_CONST False [LazyVariableTracker(unrealized: <class 'NoneType'>)]
TRACE STORE_FAST __is_tracing_resume_prologue [LazyVariableTracker(unrealized: <class 'NoneType'>), ConstantVariable(bool: False)]
TRACE JUMP_ABSOLUTE 70 [LazyVariableTracker(unrealized: <class 'NoneType'>)]
TRACE POP_TOP None [LazyVariableTracker(unrealized: <class 'NoneType'>)]
TRACE starts_line /tmp/ipykernel_503/3015389759.py:8 in torch_dynamo_resume_in_fn_at_7
        try:
TRACE SETUP_FINALLY 84 []
TRACE starts_line /tmp/ipykernel_503/3015389759.py:9 in torch_dynamo_resume_in_fn_at_7
            return x + 1
TRACE LOAD_FAST x []
TRACE LOAD_CONST 1 [LazyVariableTracker(unrealized: <class 'torch.Tensor'>)]
TRACE BINARY_ADD None [LazyVariableTracker(unrealized: <class 'torch.Tensor'>), ConstantVariable(int: 1)]
wrap_to_fake L['x'] (3,) StatefulSymbolicContext(dynamic_sizes=[<DimDynamic.STATIC: 2>], dynamic_strides=[<DimDynamic.INFER_STRIDE: 4>], constraint_sizes=[None], constraint_strides=[None], specialize_on=[[]], view_base_context=None, tensor_source=LocalSource(local_name='x', is_input=True, dynamism=None, is_derefed_cell_contents=False), shape_env_to_source_to_symbol_cache={}) <class 'torch.Tensor'>
create_graph_input L_x_ L['x'] FakeTensor(..., size=(3,)) at debug_level 0 before=False
TRACE POP_BLOCK None [TensorVariable()]
TRACE RETURN_VALUE None [TensorVariable()]
Step 1: torchdynamo done tracing torch_dynamo_resume_in_fn_at_7 (RETURN_VALUE)
return triggered compile
COMPILING GRAPH due to GraphCompileReason(reason='return_value', user_stack=[<FrameSummary file /tmp/ipykernel_503/3015389759.py, line 9 in torch_dynamo_resume_in_fn_at_7>], graph_break=False)
TRACED GRAPH
 ===== __compiled_fn_38_9f28e1b3_de53_475c_8a21_24a2170a69f5 =====
 /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/fx/_lazy_graph_module.py class GraphModule(torch.nn.Module):
    def forward(self, L_x_: "f32[3][1]cpu"):
        l_x_ = L_x_

        # File: /tmp/ipykernel_503/3015389759.py:9 in torch_dynamo_resume_in_fn_at_7, code: return x + 1
        add: "f32[3][1]cpu" = l_x_ + 1;  l_x_ = None
        return (add,)
Step 2: calling compiler function eager
Step 2: done compiler function eager
produce_guards
track_symint L['x'].size()[0] 3 None
track_symint L['x'].stride()[0] 1 None
track_symint L['x'].storage_offset() 0 None
Skipping guard L['x'].size()[0] == 3
Skipping guard L['x'].stride()[0] == 1
Skipping guard L['x'].storage_offset() == 0
GUARDS:

TREE_GUARD_MANAGER:
+- RootGuardManager
| +- LAMBDA_GUARD: torch._functorch.aot_autograd.utils.top_saved_tensors_hooks ids == None  # _dynamo/output_graph.py:866 in init_ambient_guards
| +- GLOBAL_STATE: ___check_global_state() against {"allow_bf16_reduce":0,"allow_fp16_reduce":0,"allow_tf32":false,"autocast_state":{"cached_enabled":true,"dtype":[15,5,5,15,5,5,15,15,5,5],"enabled":[false,false,false,false,false,false,false,false,false,false]},"default_dtype":6,"deterministic_algorithms":false,"deterministic_algorithms_warn_only":false,"grad_mode":true,"num_threads":4,"torch_function":true,"torch_function_all_disabled":false}
| +- TORCH_FUNCTION_MODE_STACK: ___check_torch_function_mode_stack()
| +- DEFAULT_DEVICE: utils_device.CURRENT_DEVICE == None                           # _dynamo/output_graph.py:853 in init_ambient_guards
| +- GuardManager: source=L['x'], accessed_by=FrameLocalsGuardAccessor(key='x', framelocals_idx=3), type=<class 'torch.Tensor'>, tag_safe=(True, False)
| | +- TENSOR_MATCH: check_tensor(L['x'], Tensor, DispatchKeySet(CPU, BackendSelect, ADInplaceOrView, AutogradCPU), torch.float32, device=None, requires_grad=False, size=[3], stride=[1])  # return x + 1  # mp/ipykernel_503/3015389759.py:9 in torch_dynamo_resume_in_fn_at_7
| | +- NO_HASATTR: hasattr(L['x'], '_dynamo_dynamic_indices') == False           # return x + 1  # mp/ipykernel_503/3015389759.py:9 in torch_dynamo_resume_in_fn_at_7
Guard eval latency = 436.33 us
put_code_state: no cache key, skipping
run_gc_after_compile: running gc

tensor([3.5375, 1.7539, 2.0948])

Hitting a Recompilation Limit

See Changing the Cache Size Limit.

Compiler Errors

Some compiler errors will result in skipped functions. Other compiler errors will result in a hard error rather than a skipped function.

Dealing with Skipped Functions

In general, you can resolve a skipped function by fixing the underlying graph break or error that is causing the function to be skipped.

If the graph break/error causing the skipped function is difficult to fix, then consider isolating the graph break/error in its own function so that minimal things are skipped.

def inner1(x):
    return x + 1
def inner2(x):
    return x + 2
@torch.compile
def fn(x):
    x = inner1(x)
    def problematic_code():
        torch._dynamo.skip_frame()
    problematic_code()
    x = inner2(x)
fn(torch.randn(3))

torchdynamo start compiling fn /tmp/ipykernel_503/273153676.py:5, stack (elided 5 frames):
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
    app.launch_new_instance()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
    app.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
    self.io_loop.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
    self.asyncio_loop.run_forever()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
    self._run_once()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
    handle._run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
    self._context.run(self._callback, *self._args)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
    return await f(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
    await self.dispatch_shell(msg, subshell_id=subshell_id)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
    await result
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
    await super().execute_request(stream, ident, parent)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
    reply_content = await reply_content
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
    res = shell.run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
    return super().run_cell(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
    result = self._run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
    result = runner(coro)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
    coro.send(None)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
    if await self.run_code(code, result, async_=asy):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "/tmp/ipykernel_503/273153676.py", line 12, in <module>
    fn(torch.randn(3))
Step 1: torchdynamo start tracing fn /tmp/ipykernel_503/273153676.py:5
create_env
TRACE starts_line /tmp/ipykernel_503/273153676.py:7 in fn
        x = inner1(x)
TRACE LOAD_GLOBAL inner1 []
TRACE LOAD_FAST x [LazyVariableTracker(unrealized: <class 'function'>)]
TRACE CALL_FUNCTION 1 [LazyVariableTracker(unrealized: <class 'function'>), LazyVariableTracker(unrealized: <class 'torch.Tensor'>)]
INLINING <code object inner1 at 0x7f7fc42652c0, file "/tmp/ipykernel_503/273153676.py", line 1>, inlined according trace_rules.lookup inlined by default
wrap_to_fake L['x'] (3,) StatefulSymbolicContext(dynamic_sizes=[<DimDynamic.STATIC: 2>], dynamic_strides=[<DimDynamic.INFER_STRIDE: 4>], constraint_sizes=[None], constraint_strides=[None], specialize_on=[[]], view_base_context=None, tensor_source=LocalSource(local_name='x', is_input=True, dynamism=None, is_derefed_cell_contents=False), shape_env_to_source_to_symbol_cache={}) <class 'torch.Tensor'>
create_graph_input L_x_ L['x'] FakeTensor(..., size=(3,)) at debug_level 0 before=False
TRACE starts_line /tmp/ipykernel_503/273153676.py:2 in inner1 (inline depth: 1)
        return x + 1
TRACE LOAD_FAST x []
TRACE LOAD_CONST 1 [TensorVariable()]
TRACE BINARY_ADD None [TensorVariable(), ConstantVariable(int: 1)]
TRACE RETURN_VALUE None [TensorVariable()]
DONE INLINING <code object inner1 at 0x7f7fc42652c0, file "/tmp/ipykernel_503/273153676.py", line 1>
TRACE STORE_FAST x [TensorVariable()]
TRACE starts_line /tmp/ipykernel_503/273153676.py:8 in fn
        def problematic_code():
TRACE LOAD_CONST <code object problematic_code at 0x7f7fc6909b00, file "/tmp/ipykernel_503/273153676.py", line 8> []
TRACE LOAD_CONST fn.<locals>.problematic_code [ConstantVariable(code: <code object problematic_code at 0x7f7fc6909b00, file "/tmp/ipykernel_503/273153676.py", line 8>)]
TRACE MAKE_FUNCTION 0 [ConstantVariable(code: <code object problematic_code at 0x7f7fc6909b00, file "/tmp/ipykernel_503/273153676.py", line 8>), ConstantVariable(str: 'fn.<locals>.problematic_code')]
TRACE STORE_FAST problematic_code [NestedUserFunctionVariable()]
TRACE starts_line /tmp/ipykernel_503/273153676.py:10 in fn
        problematic_code()
TRACE LOAD_FAST problematic_code []
TRACE CALL_FUNCTION 0 [NestedUserFunctionVariable()]
INLINING <code object problematic_code at 0x7f7fc6909b00, file "/tmp/ipykernel_503/273153676.py", line 8>, inlined according trace_rules.lookup inlined by default
TRACE starts_line /tmp/ipykernel_503/273153676.py:9 in problematic_code (inline depth: 1)
            torch._dynamo.skip_frame()
TRACE LOAD_GLOBAL torch []
TRACE LOAD_ATTR _dynamo [LazyVariableTracker(unrealized: <class 'module'>)]
TRACE LOAD_ATTR skip_frame [LazyVariableTracker(unrealized: <class 'module'>)]
TRACE CALL_FUNCTION 0 [LazyVariableTracker(unrealized: <class 'function'>)]
Graph break in user code at /tmp/ipykernel_503/273153676.py:9
Graph Break Reason: Encountered graph break when attempting to trace CALL_FUNCTION: a call to a regular function, e.g. f(x, y):

Call to `torch._dynamo.skip_frame()`
  Explanation: User-inserted skip frame. Message: 
  Hint: Remove the `torch._dynamo.skip_frame()` call.

  Developer debug context: Called `torch._dynamo.skip_frame()` with args `[]`, kwargs `{}`. Skipping frame problematic_code (/tmp/ipykernel_503/273153676.py line 8).

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb4297.html

User code traceback:
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
    app.launch_new_instance()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
    app.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
    self.io_loop.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
    self.asyncio_loop.run_forever()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
    self._run_once()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
    handle._run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
    self._context.run(self._callback, *self._args)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
    return await f(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
    await self.dispatch_shell(msg, subshell_id=subshell_id)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
    await result
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
    await super().execute_request(stream, ident, parent)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
    reply_content = await reply_content
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
    res = shell.run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
    return super().run_cell(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
    result = self._run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
    result = runner(coro)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
    coro.send(None)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
    if await self.run_code(code, result, async_=asy):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "/tmp/ipykernel_503/273153676.py", line 12, in <module>
    fn(torch.randn(3))
  File "/tmp/ipykernel_503/273153676.py", line 10, in fn
    problematic_code()
  File "/tmp/ipykernel_503/273153676.py", line 9, in problematic_code
    torch._dynamo.skip_frame()
Restarting analysis due to _dynamo/symbolic_convert.py:252 in fail_and_restart_analysis
Step 1: torchdynamo start tracing fn /tmp/ipykernel_503/273153676.py:5
create_env
TRACE starts_line /tmp/ipykernel_503/273153676.py:7 in fn
        x = inner1(x)
TRACE LOAD_GLOBAL inner1 []
TRACE LOAD_FAST x [LazyVariableTracker(unrealized: <class 'function'>)]
TRACE CALL_FUNCTION 1 [LazyVariableTracker(unrealized: <class 'function'>), LazyVariableTracker(unrealized: <class 'torch.Tensor'>)]
INLINING <code object inner1 at 0x7f7fc42652c0, file "/tmp/ipykernel_503/273153676.py", line 1>, inlined according trace_rules.lookup inlined by default
wrap_to_fake L['x'] (3,) StatefulSymbolicContext(dynamic_sizes=[<DimDynamic.STATIC: 2>], dynamic_strides=[<DimDynamic.INFER_STRIDE: 4>], constraint_sizes=[None], constraint_strides=[None], specialize_on=[[]], view_base_context=None, tensor_source=LocalSource(local_name='x', is_input=True, dynamism=None, is_derefed_cell_contents=False), shape_env_to_source_to_symbol_cache={}) <class 'torch.Tensor'>
create_graph_input L_x_ L['x'] FakeTensor(..., size=(3,)) at debug_level 0 before=False
TRACE starts_line /tmp/ipykernel_503/273153676.py:2 in inner1 (inline depth: 1)
        return x + 1
TRACE LOAD_FAST x []
TRACE LOAD_CONST 1 [TensorVariable()]
TRACE BINARY_ADD None [TensorVariable(), ConstantVariable(int: 1)]
TRACE RETURN_VALUE None [TensorVariable()]
DONE INLINING <code object inner1 at 0x7f7fc42652c0, file "/tmp/ipykernel_503/273153676.py", line 1>
TRACE STORE_FAST x [TensorVariable()]
TRACE starts_line /tmp/ipykernel_503/273153676.py:8 in fn
        def problematic_code():
TRACE LOAD_CONST <code object problematic_code at 0x7f7fc6909b00, file "/tmp/ipykernel_503/273153676.py", line 8> []
TRACE LOAD_CONST fn.<locals>.problematic_code [ConstantVariable(code: <code object problematic_code at 0x7f7fc6909b00, file "/tmp/ipykernel_503/273153676.py", line 8>)]
TRACE MAKE_FUNCTION 0 [ConstantVariable(code: <code object problematic_code at 0x7f7fc6909b00, file "/tmp/ipykernel_503/273153676.py", line 8>), ConstantVariable(str: 'fn.<locals>.problematic_code')]
TRACE STORE_FAST problematic_code [NestedUserFunctionVariable()]
TRACE starts_line /tmp/ipykernel_503/273153676.py:10 in fn
        problematic_code()
TRACE LOAD_FAST problematic_code []
TRACE CALL_FUNCTION 0 [NestedUserFunctionVariable()]
CALL_FUNCTION triggered compile
COMPILING GRAPH due to GraphCompileReason(reason='Call to `torch._dynamo.skip_frame()`\n  Explanation: User-inserted skip frame. Message: \n  Hint: Remove the `torch._dynamo.skip_frame()` call.\n\n  Developer debug context: Called `torch._dynamo.skip_frame()` with args `[]`, kwargs `{}`. Skipping frame problematic_code (/tmp/ipykernel_503/273153676.py line 8).\n\n For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb4297.html', user_stack=[<FrameSummary file /tmp/ipykernel_503/273153676.py, line 10 in fn>, <FrameSummary file /tmp/ipykernel_503/273153676.py, line 9 in problematic_code>], graph_break=True)
TRACED GRAPH
 ===== __compiled_fn_41_ad92873a_5630_4303_9d88_052a5699cac4 =====
 /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/fx/_lazy_graph_module.py class GraphModule(torch.nn.Module):
    def forward(self, L_x_: "f32[3][1]cpu"):
        l_x_ = L_x_

        # File: /tmp/ipykernel_503/273153676.py:2 in inner1, code: return x + 1
        x: "f32[3][1]cpu" = l_x_ + 1;  l_x_ = None
        return (x,)
Step 2: calling compiler function eager
Step 2: done compiler function eager
produce_guards
track_symint L['x'].size()[0] 3 None
track_symint L['x'].stride()[0] 1 None
track_symint L['x'].storage_offset() 0 None
Skipping guard L['x'].size()[0] == 3
Skipping guard L['x'].stride()[0] == 1
Skipping guard L['x'].storage_offset() == 0
GUARDS:

TREE_GUARD_MANAGER:
+- RootGuardManager
| +- LAMBDA_GUARD: torch._functorch.aot_autograd.utils.top_saved_tensors_hooks ids == None  # _dynamo/output_graph.py:866 in init_ambient_guards
| +- GLOBAL_STATE: ___check_global_state() against {"allow_bf16_reduce":0,"allow_fp16_reduce":0,"allow_tf32":false,"autocast_state":{"cached_enabled":true,"dtype":[15,5,5,15,5,5,15,15,5,5],"enabled":[false,false,false,false,false,false,false,false,false,false]},"default_dtype":6,"deterministic_algorithms":false,"deterministic_algorithms_warn_only":false,"grad_mode":true,"num_threads":4,"torch_function":true,"torch_function_all_disabled":false}
| +- TORCH_FUNCTION_MODE_STACK: ___check_torch_function_mode_stack()
| +- DEFAULT_DEVICE: utils_device.CURRENT_DEVICE == None                           # _dynamo/output_graph.py:853 in init_ambient_guards
| +- GuardManager: source=L['x'], accessed_by=FrameLocalsGuardAccessor(key='x', framelocals_idx=0), type=<class 'torch.Tensor'>, tag_safe=(True, False)
| | +- TENSOR_MATCH: check_tensor(L['x'], Tensor, DispatchKeySet(CPU, BackendSelect, ADInplaceOrView, AutogradCPU), torch.float32, device=None, requires_grad=False, size=[3], stride=[1])  # x = inner1(x)  # mp/ipykernel_503/273153676.py:7 in fn
| | +- NO_HASATTR: hasattr(L['x'], '_dynamo_dynamic_indices') == False           # x = inner1(x)  # mp/ipykernel_503/273153676.py:7 in fn
| +- GuardManager: source=G, accessed_by=GlobalsGuardAccessor, type=<class 'dict'>, tag_safe=(True, False)
| | +- GuardManager: source=G['inner1'], accessed_by=DictGetItemGuardAccessor('inner1'), type=<class 'function'>, tag_safe=(True, False)
| | | +- GuardManager: source=G['inner1'].__code__, accessed_by=CodeGuardAccessor, type=<class 'code'>, tag_safe=(True, False)
| | | | +- ID_MATCH: ___check_obj_id(G['inner1'].__code__, 140186728420032), type=<code object inner1 at 0x7f7fc42652c0, file "/tmp/ipykernel_503/273153676.py", line 1>  # x = inner1(x)  # mp/ipykernel_503/273153676.py:7 in fn
Guard eval latency = 328.58 us
put_code_state: no cache key, skipping
run_gc_after_compile: running gc
torchdynamo start compiling problematic_code /tmp/ipykernel_503/273153676.py:8, stack (elided 5 frames):
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
    app.launch_new_instance()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
    app.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
    self.io_loop.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
    self.asyncio_loop.run_forever()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
    self._run_once()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
    handle._run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
    self._context.run(self._callback, *self._args)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
    return await f(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
    await self.dispatch_shell(msg, subshell_id=subshell_id)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
    await result
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
    await super().execute_request(stream, ident, parent)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
    reply_content = await reply_content
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
    res = shell.run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
    return super().run_cell(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
    result = self._run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
    result = runner(coro)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
    coro.send(None)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
    if await self.run_code(code, result, async_=asy):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "/tmp/ipykernel_503/273153676.py", line 12, in <module>
    fn(torch.randn(3))
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 992, in compile_wrapper
    return fn(*args, **kwargs)
Step 1: torchdynamo start tracing problematic_code /tmp/ipykernel_503/273153676.py:8
create_env
TRACE starts_line /tmp/ipykernel_503/273153676.py:9 in problematic_code
            torch._dynamo.skip_frame()
TRACE LOAD_GLOBAL torch []
TRACE LOAD_ATTR _dynamo [LazyVariableTracker(unrealized: <class 'module'>)]
TRACE LOAD_ATTR skip_frame [LazyVariableTracker(unrealized: <class 'module'>)]
TRACE CALL_FUNCTION 0 [LazyVariableTracker(unrealized: <class 'function'>)]
Graph break (user stack suppressed due to duplicate graph break) in user code at /tmp/ipykernel_503/273153676.py:9
Graph Break Reason: Failed to handle graph break gracefully. Skipping the function and falling back to eager. Graph break encountered:

Call to `torch._dynamo.skip_frame()`
  Explanation: User-inserted skip frame. Message: 
  Hint: Remove the `torch._dynamo.skip_frame()` call.

  Developer debug context: Called `torch._dynamo.skip_frame()` with args `[]`, kwargs `{}`. Skipping frame problematic_code (/tmp/ipykernel_503/273153676.py line 8).

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb4297.html
run_gc_after_compile: running gc
WON'T CONVERT problematic_code /tmp/ipykernel_503/273153676.py line 8 
due to: 
Traceback (most recent call last):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1975, in __call__
    result = self._inner_convert(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 715, in __call__
    result = _compile(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1772, in _compile
    guarded_code, tracer_output = compile_inner(code, one_graph, hooks)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_utils_internal.py", line 96, in wrapper_function
    return function(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1445, in compile_inner
    return _compile_inner(code, one_graph, hooks)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1479, in _compile_inner
    dynamo_output = compile_frame(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1353, in compile_frame
    bytecode, tracer_output = transform_code_object(code, transform)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/bytecode_transformation.py", line 1608, in transform_code_object
    tracer_output = transformations(instructions, code_options)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1325, in transform
    tracer_output = trace_frame(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 327, in _fn
    return fn(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 849, in trace_frame
    run_tracer()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 830, in run_tracer
    tracer.run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1675, in run
    while self.step():
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1340, in step
    self.dispatch_table[inst.opcode](self, inst)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 888, in wrapper
    return inner_fn(self, inst)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 2605, in CALL_FUNCTION
    self.call_function(fn, args, {})
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1247, in call_function
    self.push(fn.call_function(self, args, kwargs))  # type: ignore[arg-type]
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/variables/lazy.py", line 433, in realize_and_forward
    return getattr(self.realize(), name)(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/variables/functions.py", line 1953, in call_function
    unimplemented(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/exc.py", line 634, in unimplemented
    raise Unsupported(msg, gb_type, skip_frame)
torch._dynamo.exc.Unsupported: Call to `torch._dynamo.skip_frame()`
  Explanation: User-inserted skip frame. Message: 
  Hint: Remove the `torch._dynamo.skip_frame()` call.

  Developer debug context: Called `torch._dynamo.skip_frame()` with args `[]`, kwargs `{}`. Skipping frame problematic_code (/tmp/ipykernel_503/273153676.py line 8).

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb4297.html

from user code:
   File "/tmp/ipykernel_503/273153676.py", line 9, in problematic_code
    torch._dynamo.skip_frame()

Set TORCHDYNAMO_VERBOSE=1 for the internal stack trace (please do this especially if you're reporting a bug to PyTorch). For even more developer context, set TORCH_LOGS="+dynamo"

Traceback (most recent call last):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1975, in __call__
    result = self._inner_convert(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 715, in __call__
    result = _compile(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1772, in _compile
    guarded_code, tracer_output = compile_inner(code, one_graph, hooks)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_utils_internal.py", line 96, in wrapper_function
    return function(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1445, in compile_inner
    return _compile_inner(code, one_graph, hooks)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1479, in _compile_inner
    dynamo_output = compile_frame(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1353, in compile_frame
    bytecode, tracer_output = transform_code_object(code, transform)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/bytecode_transformation.py", line 1608, in transform_code_object
    tracer_output = transformations(instructions, code_options)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1325, in transform
    tracer_output = trace_frame(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 327, in _fn
    return fn(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 849, in trace_frame
    run_tracer()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 830, in run_tracer
    tracer.run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1675, in run
    while self.step():
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1340, in step
    self.dispatch_table[inst.opcode](self, inst)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 888, in wrapper
    return inner_fn(self, inst)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 2605, in CALL_FUNCTION
    self.call_function(fn, args, {})
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1247, in call_function
    self.push(fn.call_function(self, args, kwargs))  # type: ignore[arg-type]
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/variables/lazy.py", line 433, in realize_and_forward
    return getattr(self.realize(), name)(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/variables/functions.py", line 1953, in call_function
    unimplemented(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/exc.py", line 634, in unimplemented
    raise Unsupported(msg, gb_type, skip_frame)
torch._dynamo.exc.Unsupported: Call to `torch._dynamo.skip_frame()`
  Explanation: User-inserted skip frame. Message: 
  Hint: Remove the `torch._dynamo.skip_frame()` call.

  Developer debug context: Called `torch._dynamo.skip_frame()` with args `[]`, kwargs `{}`. Skipping frame problematic_code (/tmp/ipykernel_503/273153676.py line 8).

 For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb4297.html

from user code:
   File "/tmp/ipykernel_503/273153676.py", line 9, in problematic_code
    torch._dynamo.skip_frame()

Set TORCHDYNAMO_VERBOSE=1 for the internal stack trace (please do this especially if you're reporting a bug to PyTorch). For even more developer context, set TORCH_LOGS="+dynamo"
torchdynamo start compiling torch_dynamo_resume_in_fn_at_10 /tmp/ipykernel_503/273153676.py:10, stack (elided 5 frames):
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/opt/conda/envs/py_3.10/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel_launcher.py", line 18, in <module>
    app.launch_new_instance()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
    app.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 758, in start
    self.io_loop.start()
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 211, in start
    self.asyncio_loop.run_forever()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
    self._run_once()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
    handle._run()
  File "/opt/conda/envs/py_3.10/lib/python3.10/asyncio/events.py", line 80, in _run
    self._context.run(self._callback, *self._args)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/utils.py", line 71, in preserve_context
    return await f(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 614, in shell_main
    await self.dispatch_shell(msg, subshell_id=subshell_id)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 471, in dispatch_shell
    await result
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 366, in execute_request
    await super().execute_request(stream, ident, parent)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 827, in execute_request
    reply_content = await reply_content
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 458, in do_execute
    res = shell.run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 663, in run_cell
    return super().run_cell(*args, **kwargs)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
    result = self._run_cell(
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
    result = runner(coro)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
    coro.send(None)
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
    if await self.run_code(code, result, async_=asy):
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "/tmp/ipykernel_503/273153676.py", line 12, in <module>
    fn(torch.randn(3))
  File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 992, in compile_wrapper
    return fn(*args, **kwargs)
Step 1: torchdynamo start tracing torch_dynamo_resume_in_fn_at_10 /tmp/ipykernel_503/273153676.py:10
create_env
TRACE starts_line /tmp/ipykernel_503/273153676.py:10 in torch_dynamo_resume_in_fn_at_10
        problematic_code()
TRACE LOAD_CONST True []
TRACE STORE_FAST __is_tracing_resume_prologue [ConstantVariable(bool: True)]
TRACE LOAD_FAST ___stack0 []
TRACE LOAD_CONST False [LazyVariableTracker(unrealized: <class 'NoneType'>)]
TRACE STORE_FAST __is_tracing_resume_prologue [LazyVariableTracker(unrealized: <class 'NoneType'>), ConstantVariable(bool: False)]
TRACE JUMP_ABSOLUTE 32 [LazyVariableTracker(unrealized: <class 'NoneType'>)]
TRACE POP_TOP None [LazyVariableTracker(unrealized: <class 'NoneType'>)]
TRACE starts_line /tmp/ipykernel_503/273153676.py:11 in torch_dynamo_resume_in_fn_at_10
        x = inner2(x)
TRACE LOAD_GLOBAL inner2 []
TRACE LOAD_FAST x [LazyVariableTracker(unrealized: <class 'function'>)]
TRACE CALL_FUNCTION 1 [LazyVariableTracker(unrealized: <class 'function'>), LazyVariableTracker(unrealized: <class 'torch.Tensor'>)]
INLINING <code object inner2 at 0x7f7f5decaef0, file "/tmp/ipykernel_503/273153676.py", line 3>, inlined according trace_rules.lookup inlined by default
wrap_to_fake L['x'] (3,) StatefulSymbolicContext(dynamic_sizes=[<DimDynamic.STATIC: 2>], dynamic_strides=[<DimDynamic.INFER_STRIDE: 4>], constraint_sizes=[None], constraint_strides=[None], specialize_on=[[]], view_base_context=None, tensor_source=LocalSource(local_name='x', is_input=True, dynamism=None, is_derefed_cell_contents=False), shape_env_to_source_to_symbol_cache={}) <class 'torch.Tensor'>
create_graph_input L_x_ L['x'] FakeTensor(..., size=(3,)) at debug_level 0 before=False
TRACE starts_line /tmp/ipykernel_503/273153676.py:4 in inner2 (inline depth: 1)
        return x + 2
TRACE LOAD_FAST x []
TRACE LOAD_CONST 2 [TensorVariable()]
TRACE BINARY_ADD None [TensorVariable(), ConstantVariable(int: 2)]
TRACE RETURN_VALUE None [TensorVariable()]
DONE INLINING <code object inner2 at 0x7f7f5decaef0, file "/tmp/ipykernel_503/273153676.py", line 3>
TRACE STORE_FAST x [TensorVariable()]
TRACE LOAD_CONST None []
TRACE RETURN_VALUE None [ConstantVariable(NoneType: None)]
Step 1: torchdynamo done tracing torch_dynamo_resume_in_fn_at_10 (RETURN_VALUE)
return triggered compile
COMPILING GRAPH due to GraphCompileReason(reason='return_value', user_stack=[<FrameSummary file /tmp/ipykernel_503/273153676.py, line 11 in torch_dynamo_resume_in_fn_at_10>], graph_break=False)
TRACED GRAPH
 ===== __compiled_fn_45_a319514c_bcda_4a87_8a4a_37cf6ab6f726 =====
 /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/fx/_lazy_graph_module.py class GraphModule(torch.nn.Module):
    def forward(self, L_x_: "f32[3][1]cpu"):
        l_x_ = L_x_

        # File: /tmp/ipykernel_503/273153676.py:4 in inner2, code: return x + 2
        x: "f32[3][1]cpu" = l_x_ + 2;  l_x_ = x = None
        return ()
Step 2: calling compiler function eager
Step 2: done compiler function eager
produce_guards
track_symint L['x'].size()[0] 3 None
track_symint L['x'].stride()[0] 1 None
track_symint L['x'].storage_offset() 0 None
Skipping guard L['x'].size()[0] == 3
Skipping guard L['x'].stride()[0] == 1
Skipping guard L['x'].storage_offset() == 0
GUARDS:

TREE_GUARD_MANAGER:
+- RootGuardManager
| +- LAMBDA_GUARD: torch._functorch.aot_autograd.utils.top_saved_tensors_hooks ids == None  # _dynamo/output_graph.py:866 in init_ambient_guards
| +- GLOBAL_STATE: ___check_global_state() against {"allow_bf16_reduce":0,"allow_fp16_reduce":0,"allow_tf32":false,"autocast_state":{"cached_enabled":true,"dtype":[15,5,5,15,5,5,15,15,5,5],"enabled":[false,false,false,false,false,false,false,false,false,false]},"default_dtype":6,"deterministic_algorithms":false,"deterministic_algorithms_warn_only":false,"grad_mode":true,"num_threads":4,"torch_function":true,"torch_function_all_disabled":false}
| +- TORCH_FUNCTION_MODE_STACK: ___check_torch_function_mode_stack()
| +- DEFAULT_DEVICE: utils_device.CURRENT_DEVICE == None                           # _dynamo/output_graph.py:853 in init_ambient_guards
| +- GuardManager: source=L['x'], accessed_by=FrameLocalsGuardAccessor(key='x', framelocals_idx=3), type=<class 'torch.Tensor'>, tag_safe=(True, False)
| | +- TENSOR_MATCH: check_tensor(L['x'], Tensor, DispatchKeySet(CPU, BackendSelect, ADInplaceOrView, AutogradCPU), torch.float32, device=None, requires_grad=False, size=[3], stride=[1])  # x = inner2(x)  # mp/ipykernel_503/273153676.py:11 in torch_dynamo_resume_in_fn_at_10
| | +- NO_HASATTR: hasattr(L['x'], '_dynamo_dynamic_indices') == False           # x = inner2(x)  # mp/ipykernel_503/273153676.py:11 in torch_dynamo_resume_in_fn_at_10
| +- GuardManager: source=G, accessed_by=GlobalsGuardAccessor, type=<class 'dict'>, tag_safe=(True, False)
| | +- GuardManager: source=G['inner2'], accessed_by=DictGetItemGuardAccessor('inner2'), type=<class 'function'>, tag_safe=(True, False)
| | | +- GuardManager: source=G['inner2'].__code__, accessed_by=CodeGuardAccessor, type=<class 'code'>, tag_safe=(True, False)
| | | | +- ID_MATCH: ___check_obj_id(G['inner2'].__code__, 140185013366512), type=<code object inner2 at 0x7f7f5decaef0, file "/tmp/ipykernel_503/273153676.py", line 3>  # x = inner2(x)  # mp/ipykernel_503/273153676.py:11 in torch_dynamo_resume_in_fn_at_10
Guard eval latency = 307.03 us
put_code_state: no cache key, skipping
run_gc_after_compile: running gc