Generate Python docs from pytorch/pytorch@6c8c5ad

Author: pytorchbot

docs/2.2/_images/RReLU.png

@@ -1126,7 +1126,13 @@ <h1>Source code for torch</h1><div class="highlight"><pre>
 
 
 <div class="viewcode-block" id="set_default_tensor_type"><a class="viewcode-back" href="../generated/torch.set_default_tensor_type.html#torch.set_default_tensor_type">[docs]</a><span class="k">def</span> <span class="nf">set_default_tensor_type</span><span class="p">(</span><span class="n">t</span><span class="p">):</span>
-<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Sets the default ``torch.Tensor`` type to floating point tensor type</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    .. warning::</span>
+
+<span class="sd">        This function is deprecated as of PyTorch 2.1, please use :func:`torch.set_default_dtype()` and</span>
+<span class="sd">        :func:`torch.set_default_device()` as alternatives.</span>
+
+<span class="sd">    Sets the default ``torch.Tensor`` type to floating point tensor type</span>
 <span class="sd">    ``t``. This type will also be used as default floating point type for</span>
 <span class="sd">    type inference in :func:`torch.tensor`.</span>
 

docs/2.2/_modules/torch.html

@@ -621,7 +621,7 @@ <h1>Source code for torch.distributed.checkpoint.state_dict</h1><div class="high
             <span class="k">if</span> <span class="ow">not</span> <span class="n">skip_ddp_prefix</span><span class="p">:</span>
                 <span class="n">fqn_obj_names</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">curr_obj_name</span><span class="p">)</span>
         <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">curr_obj</span><span class="p">,</span> <span class="n">FSDP</span><span class="p">):</span>
-            <span class="k">if</span> <span class="n">obj_names</span><span class="p">[</span><span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">]</span> <span class="o">==</span> <span class="n">FLAT_PARAM</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">i</span> <span class="o">&lt;</span> <span class="nb">len</span><span class="p">(</span><span class="n">obj_names</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span> <span class="ow">and</span> <span class="n">obj_names</span><span class="p">[</span><span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">]</span> <span class="o">==</span> <span class="n">FLAT_PARAM</span><span class="p">:</span>
                 <span class="n">prefix</span> <span class="o">=</span> <span class="s2">&quot;.&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">fqn_obj_names</span><span class="p">)</span>
                 <span class="n">flat_param</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">curr_obj</span><span class="p">,</span> <span class="n">FLAT_PARAM</span><span class="p">)</span>
                 <span class="k">if</span> <span class="n">prefix</span><span class="p">:</span>
@@ -660,7 +660,7 @@ <h1>Source code for torch.distributed.checkpoint.state_dict</h1><div class="high
         <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">],</span> <span class="n">Union</span><span class="p">[</span><span class="n">Set</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">]</span>
     <span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
     <span class="n">all_fqns</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
-    <span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">param</span> <span class="ow">in</span> <span class="n">model</span><span class="o">.</span><span class="n">named_parameters</span><span class="p">():</span>
+    <span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">param</span> <span class="ow">in</span> <span class="n">chain</span><span class="p">(</span><span class="n">model</span><span class="o">.</span><span class="n">named_parameters</span><span class="p">(),</span> <span class="n">model</span><span class="o">.</span><span class="n">named_buffers</span><span class="p">()):</span>
         <span class="n">fqns</span> <span class="o">=</span> <span class="n">_get_fqns</span><span class="p">(</span><span class="n">model</span><span class="p">,</span> <span class="n">name</span><span class="p">)</span>
         <span class="n">fqn_param_mapping</span><span class="p">[</span><span class="n">param</span><span class="p">]</span> <span class="o">=</span> <span class="n">fqns</span>
         <span class="k">for</span> <span class="n">fqn</span> <span class="ow">in</span> <span class="n">fqns</span><span class="p">:</span>
@@ -859,7 +859,7 @@ <h1>Source code for torch.distributed.checkpoint.state_dict</h1><div class="high
     <span class="k">if</span> <span class="ow">not</span> <span class="n">info</span><span class="o">.</span><span class="n">handle_model</span> <span class="ow">or</span> <span class="ow">not</span> <span class="n">state_dict</span><span class="p">:</span>
         <span class="k">return</span> <span class="n">_IncompatibleKeys</span><span class="p">({},</span> <span class="p">{})</span>
 
-    <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">_</span> <span class="ow">in</span> <span class="n">model</span><span class="o">.</span><span class="n">named_parameters</span><span class="p">():</span>
+    <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">_</span> <span class="ow">in</span> <span class="n">chain</span><span class="p">(</span><span class="n">model</span><span class="o">.</span><span class="n">named_parameters</span><span class="p">(),</span> <span class="n">model</span><span class="o">.</span><span class="n">named_buffers</span><span class="p">()):</span>
         <span class="n">fqns</span> <span class="o">=</span> <span class="n">_get_fqns</span><span class="p">(</span><span class="n">model</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span>
         <span class="n">fqns_with_ddp_prefix</span> <span class="o">=</span> <span class="n">_get_fqns</span><span class="p">(</span><span class="n">model</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">skip_ddp_prefix</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
         <span class="k">for</span> <span class="n">fqn</span><span class="p">,</span> <span class="n">fqn_with_ddp_prefix</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">fqns</span><span class="p">,</span> <span class="n">fqns_with_ddp_prefix</span><span class="p">):</span>
@@ -1142,25 +1142,25 @@ <h1>Source code for torch.distributed.checkpoint.state_dict</h1><div class="high
 <span class="sd">    optimizer parameter IDs to the canonical FQNs.</span>
 
 <span class="sd">    Example:</span>
+<span class="sd">        &gt;&gt;&gt; # xdoctest: +SKIP</span>
+<span class="sd">        &gt;&gt;&gt; import torch</span>
+<span class="sd">        &gt;&gt;&gt; from torch.distributed.fsdp import FullyShardedDataParallel as FSDP</span>
+<span class="sd">        &gt;&gt;&gt; from torch.nn.parallel import DistributedDataParallel as DDP</span>
+<span class="sd">        &gt;&gt;&gt; from torch.distributed.checkpoint.state_dict import get_state_dict</span>
 
-<span class="sd">        import torch</span>
-<span class="sd">        from torch.distributed.fsdp import FullyShardedDataParallel as FSDP</span>
-<span class="sd">        from torch.nn.parallel import DistributedDataParallel as DDP</span>
-<span class="sd">        from torch.distributed.checkpoint.state_dict import get_state_dict</span>
-
-<span class="sd">        fsdp_model = FSDP(copy.deepcopy(model))</span>
-<span class="sd">        fsdp_optim = torch.optim.Adam(model.parameters(), lr=1e-3)</span>
-<span class="sd">        ddp_model = DDP(copy.deepcopy(model))</span>
-<span class="sd">        ddp_optim = torch.optim.Adam(model.parameters(), lr=1e-3)</span>
+<span class="sd">        &gt;&gt;&gt; fsdp_model = FSDP(copy.deepcopy(model))</span>
+<span class="sd">        &gt;&gt;&gt; fsdp_optim = torch.optim.Adam(model.parameters(), lr=1e-3)</span>
+<span class="sd">        &gt;&gt;&gt; ddp_model = DDP(copy.deepcopy(model))</span>
+<span class="sd">        &gt;&gt;&gt; ddp_optim = torch.optim.Adam(model.parameters(), lr=1e-3)</span>
 
 
-<span class="sd">        ddp_state_dict, ddp_optim_state_dict = get_state_dict(ddp_model, ddp_optim)</span>
-<span class="sd">        fsdp_state_dict, fsdp_optim_state_dict = get_state_dict(fsdp_model, fsdp_optim)</span>
+<span class="sd">        &gt;&gt;&gt; ddp_state_dict, ddp_optim_state_dict = get_state_dict(ddp_model, ddp_optim)</span>
+<span class="sd">        &gt;&gt;&gt; fsdp_state_dict, fsdp_optim_state_dict = get_state_dict(fsdp_model, fsdp_optim)</span>
 
-<span class="sd">        # if we simply call ddp_model.state_dict() and fsdp_model.state_dict(),</span>
-<span class="sd">        # the asserts will fail.</span>
-<span class="sd">        assert ddp_state_dict == fsdp_state_dict</span>
-<span class="sd">        assert ddp_optim_state == fsdp_optim_state_dict</span>
+<span class="sd">        &gt;&gt;&gt; # if we simply call ddp_model.state_dict() and fsdp_model.state_dict(),</span>
+<span class="sd">        &gt;&gt;&gt; # the asserts will fail.</span>
+<span class="sd">        &gt;&gt;&gt; assert ddp_state_dict == fsdp_state_dict</span>
+<span class="sd">        &gt;&gt;&gt; assert ddp_optim_state == fsdp_optim_state_dict</span>
 
 
 <span class="sd">    Args:</span>
@@ -1175,6 +1175,8 @@ <h1>Source code for torch.distributed.checkpoint.state_dict</h1><div class="high
 
 <span class="sd">    Returns:</span>
 <span class="sd">        ``Tuple`` that contain model state_dict and optimizer state_dict.</span>
+
+<span class="sd">    :rtype: typing.Tuple[typing.Dict[str, ValueType], OptimizerStateType]</span>
 <span class="sd">    &quot;&quot;&quot;</span>
 
     <span class="k">with</span> <span class="n">gc_context</span><span class="p">():</span>

docs/2.2/_modules/torch/distributed/checkpoint/state_dict.html

@@ -932,7 +932,7 @@ <h1>Source code for torch.distributed.fsdp.fully_sharded_data_parallel</h1><div
                 <span class="s2">&quot;ignored_states&quot;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_ignored_params</span><span class="p">,</span>
                 <span class="s2">&quot;device_mesh&quot;</span><span class="p">:</span> <span class="n">device_mesh</span><span class="p">,</span>
             <span class="p">}</span>
-            <span class="k">if</span> <span class="n">sharding_strategy</span> <span class="ow">in</span> <span class="n">HYBRID_SHARDING_STRATEGIES</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">sharding_strategy</span> <span class="ow">in</span> <span class="n">HYBRID_SHARDING_STRATEGIES</span> <span class="ow">and</span> <span class="n">device_mesh</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
                 <span class="c1"># Share root process groups with children to maintain</span>
                 <span class="c1"># the invariant that all FSDP modules will have the same</span>
                 <span class="c1"># process groups.</span>

docs/2.2/_modules/torch/distributed/fsdp/fully_sharded_data_parallel.html

@@ -495,7 +495,7 @@ <h1>Source code for torch.distributed.tensor.parallel.style</h1><div class="high
 
 <div class="viewcode-block" id="ColwiseParallel"><a class="viewcode-back" href="../../../../../distributed.tensor.parallel.html#torch.distributed.tensor.parallel.ColwiseParallel">[docs]</a><span class="k">class</span> <span class="nc">ColwiseParallel</span><span class="p">(</span><span class="n">ParallelStyle</span><span class="p">):</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">    Partition a compatible nn.Module in a row-wise fashion. Currently supports nn.Linear and nn.Embedding.</span>
+<span class="sd">    Partition a compatible nn.Module in a column-wise fashion. Currently supports nn.Linear and nn.Embedding.</span>
 <span class="sd">    Users can compose it together with RowwiseParallel to achieve the sharding of more complicated modules.</span>
 <span class="sd">    (i.e. MLP, Attention)</span>
 

docs/2.2/_modules/torch/distributed/tensor/parallel/style.html

@@ -491,7 +491,7 @@ <h1>Source code for torch.nn.modules.padding</h1><div class="highlight"><pre>
         <span class="k">return</span> <span class="sa">f</span><span class="s1">&#39;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">padding</span><span class="si">}</span><span class="s1">&#39;</span>
 
 
-<span class="k">class</span> <span class="nc">CircularPad1d</span><span class="p">(</span><span class="n">_CircularPadNd</span><span class="p">):</span>
+<div class="viewcode-block" id="CircularPad1d"><a class="viewcode-back" href="../../../../generated/torch.nn.CircularPad1d.html#torch.nn.CircularPad1d">[docs]</a><span class="k">class</span> <span class="nc">CircularPad1d</span><span class="p">(</span><span class="n">_CircularPadNd</span><span class="p">):</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Pads the input tensor using circular padding of the input boundary.</span>
 
 <span class="sd">    Tensor values at the beginning of the dimension are used to pad the end,</span>
@@ -539,10 +539,10 @@ <h1>Source code for torch.nn.modules.padding</h1><div class="highlight"><pre>
         <span class="k">if</span> <span class="nb">input</span><span class="o">.</span><span class="n">dim</span><span class="p">()</span> <span class="o">!=</span> <span class="mi">2</span> <span class="ow">and</span> <span class="nb">input</span><span class="o">.</span><span class="n">dim</span><span class="p">()</span> <span class="o">!=</span> <span class="mi">3</span><span class="p">:</span>
             <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
                 <span class="sa">f</span><span class="s2">&quot;expected 2D or 3D input (got </span><span class="si">{</span><span class="nb">input</span><span class="o">.</span><span class="n">dim</span><span class="p">()</span><span class="si">}</span><span class="s2">D input)&quot;</span>
-            <span class="p">)</span>
+            <span class="p">)</span></div>
 
 
-<span class="k">class</span> <span class="nc">CircularPad2d</span><span class="p">(</span><span class="n">_CircularPadNd</span><span class="p">):</span>
+<div class="viewcode-block" id="CircularPad2d"><a class="viewcode-back" href="../../../../generated/torch.nn.CircularPad2d.html#torch.nn.CircularPad2d">[docs]</a><span class="k">class</span> <span class="nc">CircularPad2d</span><span class="p">(</span><span class="n">_CircularPadNd</span><span class="p">):</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Pads the input tensor using circular padding of the input boundary.</span>
 
 <span class="sd">    Tensor values at the beginning of the dimension are used to pad the end,</span>
@@ -600,10 +600,10 @@ <h1>Source code for torch.nn.modules.padding</h1><div class="highlight"><pre>
         <span class="k">if</span> <span class="nb">input</span><span class="o">.</span><span class="n">dim</span><span class="p">()</span> <span class="o">!=</span> <span class="mi">3</span> <span class="ow">and</span> <span class="nb">input</span><span class="o">.</span><span class="n">dim</span><span class="p">()</span> <span class="o">!=</span> <span class="mi">4</span><span class="p">:</span>
             <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
                 <span class="sa">f</span><span class="s2">&quot;expected 3D or 4D input (got </span><span class="si">{</span><span class="nb">input</span><span class="o">.</span><span class="n">dim</span><span class="p">()</span><span class="si">}</span><span class="s2">D input)&quot;</span>
-            <span class="p">)</span>
+            <span class="p">)</span></div>
 
 
-<span class="k">class</span> <span class="nc">CircularPad3d</span><span class="p">(</span><span class="n">_CircularPadNd</span><span class="p">):</span>
+<div class="viewcode-block" id="CircularPad3d"><a class="viewcode-back" href="../../../../generated/torch.nn.CircularPad3d.html#torch.nn.CircularPad3d">[docs]</a><span class="k">class</span> <span class="nc">CircularPad3d</span><span class="p">(</span><span class="n">_CircularPadNd</span><span class="p">):</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Pads the input tensor using circular padding of the input boundary.</span>
 
 <span class="sd">    Tensor values at the beginning of the dimension are used to pad the end,</span>
@@ -651,7 +651,7 @@ <h1>Source code for torch.nn.modules.padding</h1><div class="highlight"><pre>
         <span class="k">if</span> <span class="nb">input</span><span class="o">.</span><span class="n">dim</span><span class="p">()</span> <span class="o">!=</span> <span class="mi">4</span> <span class="ow">and</span> <span class="nb">input</span><span class="o">.</span><span class="n">dim</span><span class="p">()</span> <span class="o">!=</span> <span class="mi">5</span><span class="p">:</span>
             <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
                 <span class="sa">f</span><span class="s2">&quot;expected 4D or 5D input (got </span><span class="si">{</span><span class="nb">input</span><span class="o">.</span><span class="n">dim</span><span class="p">()</span><span class="si">}</span><span class="s2">D input)&quot;</span>
-            <span class="p">)</span>
+            <span class="p">)</span></div>
 
 
 <span class="k">class</span> <span class="nc">_ConstantPadNd</span><span class="p">(</span><span class="n">Module</span><span class="p">):</span>