deploy: 86b4ab4

Author: github-actions[bot]

api/_modules/opacus/grad_sample/grad_sample_module_fast_gradient_clipping.html

@@ -53,7 +53,12 @@ <h1>Source code for opacus.grad_sample.grad_sample_module_fast_gradient_clipping
     <span class="n">create_or_accumulate_grad_sample</span><span class="p">,</span>
     <span class="n">promote_current_grad_sample</span><span class="p">,</span>
 <span class="p">)</span>
-<span class="kn">from</span> <span class="nn">opacus.utils.module_utils</span> <span class="kn">import</span> <span class="n">requires_grad</span><span class="p">,</span> <span class="n">trainable_parameters</span>
+<span class="kn">from</span> <span class="nn">opacus.layers.dp_rnn</span> <span class="kn">import</span> <span class="n">DPGRU</span><span class="p">,</span> <span class="n">DPLSTM</span><span class="p">,</span> <span class="n">DPRNN</span>
+<span class="kn">from</span> <span class="nn">opacus.utils.module_utils</span> <span class="kn">import</span> <span class="p">(</span>
+    <span class="n">requires_grad</span><span class="p">,</span>
+    <span class="n">trainable_modules</span><span class="p">,</span>
+    <span class="n">trainable_parameters</span><span class="p">,</span>
+<span class="p">)</span>
 
 
 <span class="n">logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="vm">__name__</span><span class="p">)</span>
@@ -141,6 +146,12 @@ <h1>Source code for opacus.grad_sample.grad_sample_module_fast_gradient_clipping
 <span class="sd">                If ``strict`` is set to ``True`` and module ``m`` (or any of its</span>
 <span class="sd">                submodules) includes a buffer.</span>
 <span class="sd">        """</span>
+        <span class="k">if</span> <span class="n">logger</span><span class="o">.</span><span class="n">isEnabledFor</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">INFO</span><span class="p">):</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log_module_gradient_sample_mode</span><span class="p">(</span>
+                <span class="n">module</span><span class="o">=</span><span class="n">m</span><span class="p">,</span>
+                <span class="n">force_functorch</span><span class="o">=</span><span class="n">force_functorch</span><span class="p">,</span>
+                <span class="n">use_ghost_clipping</span><span class="o">=</span><span class="n">use_ghost_clipping</span><span class="p">,</span>
+            <span class="p">)</span>
 
         <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
             <span class="n">m</span><span class="p">,</span>
@@ -275,6 +286,46 @@ <h1>Source code for opacus.grad_sample.grad_sample_module_fast_gradient_clipping
                 <span class="k">del</span> <span class="n">module</span><span class="o">.</span><span class="n">max_batch_len</span></div>
 
 
+<div class="viewcode-block" id="GradSampleModuleFastGradientClipping.log_module_gradient_sample_mode">
+<a class="viewcode-back" href="../../../grad_sample_module_fast_gradient_clipping.html#opacus.grad_sample.grad_sample_module_fast_gradient_clipping.GradSampleModuleFastGradientClipping.log_module_gradient_sample_mode">[docs]</a>
+    <span class="k">def</span> <span class="nf">log_module_gradient_sample_mode</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span> <span class="n">module</span><span class="p">:</span> <span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">force_functorch</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">use_ghost_clipping</span><span class="o">=</span><span class="kc">True</span>
+    <span class="p">):</span>
+<span class="w">        </span><span class="sd">"""</span>
+<span class="sd">        Add logs to track gradient sample mode for each part of the module, including 1) Ghost Clipping, 2) Fast Gradient Clipping (hook mode), and 3) Fast Gradient Clipping (functorch mode).</span>
+
+<span class="sd">        Args:</span>
+<span class="sd">            module: nn.Module to be checked</span>
+<span class="sd">            force_functorch: If set to ``True``, will use functorch to compute</span>
+<span class="sd">                all per sample gradients. Otherwise, functorch will be used only</span>
+<span class="sd">                for layers without registered grad sampler methods.</span>
+<span class="sd">            use_ghost_clipping: If set to ``True``, Ghost Clipping</span>
+<span class="sd">                will be used for clipping gradients of supported layers. If ``False``, Fast</span>
+<span class="sd">                Gradient Clipping will be used for all layers.</span>
+<span class="sd">        """</span>
+        <span class="k">for</span> <span class="n">m_name</span><span class="p">,</span> <span class="n">m</span> <span class="ow">in</span> <span class="n">trainable_modules</span><span class="p">(</span><span class="n">module</span><span class="p">):</span>
+            <span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">m</span><span class="p">)</span> <span class="ow">in</span> <span class="p">[</span><span class="n">DPRNN</span><span class="p">,</span> <span class="n">DPLSTM</span><span class="p">,</span> <span class="n">DPGRU</span><span class="p">]:</span>
+                <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                    <span class="sa">f</span><span class="s2">"Module name: </span><span class="si">{</span><span class="n">m_name</span><span class="si">}</span><span class="s2">, module type: </span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">m</span><span class="p">)</span><span class="si">}</span><span class="s2">. No hook or functorch is added."</span>
+                <span class="p">)</span>
+
+            <span class="k">elif</span> <span class="n">use_ghost_clipping</span> <span class="ow">and</span> <span class="nb">type</span><span class="p">(</span><span class="n">m</span><span class="p">)</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">NORM_SAMPLERS</span><span class="p">:</span>
+                <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                    <span class="sa">f</span><span class="s2">"Module name: </span><span class="si">{</span><span class="n">m_name</span><span class="si">}</span><span class="s2">, module type: </span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">m</span><span class="p">)</span><span class="si">}</span><span class="s2">, under Ghost Clipping."</span>
+                <span class="p">)</span>
+
+            <span class="k">else</span><span class="p">:</span>
+                <span class="k">if</span> <span class="ow">not</span> <span class="n">force_functorch</span> <span class="ow">and</span> <span class="nb">type</span><span class="p">(</span><span class="n">m</span><span class="p">)</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">GRAD_SAMPLERS</span><span class="p">:</span>
+                    <span class="c1"># When functorch is not enforced, use FGC (hook mode) if the layer has a registered grad_sampler (supported). Otherwise, use FGC (functorch mode).</span>
+                    <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                        <span class="sa">f</span><span class="s2">"Module name: </span><span class="si">{</span><span class="n">m_name</span><span class="si">}</span><span class="s2">, module type: </span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">m</span><span class="p">)</span><span class="si">}</span><span class="s2">, under Fast Gradient Clipping (hook mode)."</span>
+                    <span class="p">)</span>
+                <span class="k">else</span><span class="p">:</span>
+                    <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                        <span class="sa">f</span><span class="s2">"Module name: </span><span class="si">{</span><span class="n">m_name</span><span class="si">}</span><span class="s2">, module type: </span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">m</span><span class="p">)</span><span class="si">}</span><span class="s2">, under Fast Gradient Clipping (functorch mode)."</span>
+                    <span class="p">)</span></div>
+
+
     <span class="nd">@property</span>
     <span class="k">def</span> <span class="nf">per_sample_gradient_norms</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">:</span>
 <span class="w">        </span><span class="sd">"""Returns per sample gradient norms. Note that these are not privatized and should only be used for debugging purposes or in non-private settings"""</span>

api/_modules/opacus/grad_sample/grad_sample_module_fast_gradient_clipping/index.html

@@ -53,7 +53,12 @@ <h1>Source code for opacus.grad_sample.grad_sample_module_fast_gradient_clipping
     <span class="n">create_or_accumulate_grad_sample</span><span class="p">,</span>
     <span class="n">promote_current_grad_sample</span><span class="p">,</span>
 <span class="p">)</span>
-<span class="kn">from</span> <span class="nn">opacus.utils.module_utils</span> <span class="kn">import</span> <span class="n">requires_grad</span><span class="p">,</span> <span class="n">trainable_parameters</span>
+<span class="kn">from</span> <span class="nn">opacus.layers.dp_rnn</span> <span class="kn">import</span> <span class="n">DPGRU</span><span class="p">,</span> <span class="n">DPLSTM</span><span class="p">,</span> <span class="n">DPRNN</span>
+<span class="kn">from</span> <span class="nn">opacus.utils.module_utils</span> <span class="kn">import</span> <span class="p">(</span>
+    <span class="n">requires_grad</span><span class="p">,</span>
+    <span class="n">trainable_modules</span><span class="p">,</span>
+    <span class="n">trainable_parameters</span><span class="p">,</span>
+<span class="p">)</span>
 
 
 <span class="n">logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="vm">__name__</span><span class="p">)</span>
@@ -141,6 +146,12 @@ <h1>Source code for opacus.grad_sample.grad_sample_module_fast_gradient_clipping
 <span class="sd">                If ``strict`` is set to ``True`` and module ``m`` (or any of its</span>
 <span class="sd">                submodules) includes a buffer.</span>
 <span class="sd">        """</span>
+        <span class="k">if</span> <span class="n">logger</span><span class="o">.</span><span class="n">isEnabledFor</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">INFO</span><span class="p">):</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log_module_gradient_sample_mode</span><span class="p">(</span>
+                <span class="n">module</span><span class="o">=</span><span class="n">m</span><span class="p">,</span>
+                <span class="n">force_functorch</span><span class="o">=</span><span class="n">force_functorch</span><span class="p">,</span>
+                <span class="n">use_ghost_clipping</span><span class="o">=</span><span class="n">use_ghost_clipping</span><span class="p">,</span>
+            <span class="p">)</span>
 
         <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
             <span class="n">m</span><span class="p">,</span>
@@ -275,6 +286,46 @@ <h1>Source code for opacus.grad_sample.grad_sample_module_fast_gradient_clipping
                 <span class="k">del</span> <span class="n">module</span><span class="o">.</span><span class="n">max_batch_len</span></div>
 
 
+<div class="viewcode-block" id="GradSampleModuleFastGradientClipping.log_module_gradient_sample_mode">
+<a class="viewcode-back" href="../../../grad_sample_module_fast_gradient_clipping.html#opacus.grad_sample.grad_sample_module_fast_gradient_clipping.GradSampleModuleFastGradientClipping.log_module_gradient_sample_mode">[docs]</a>
+    <span class="k">def</span> <span class="nf">log_module_gradient_sample_mode</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span> <span class="n">module</span><span class="p">:</span> <span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">force_functorch</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">use_ghost_clipping</span><span class="o">=</span><span class="kc">True</span>
+    <span class="p">):</span>
+<span class="w">        </span><span class="sd">"""</span>
+<span class="sd">        Add logs to track gradient sample mode for each part of the module, including 1) Ghost Clipping, 2) Fast Gradient Clipping (hook mode), and 3) Fast Gradient Clipping (functorch mode).</span>
+
+<span class="sd">        Args:</span>
+<span class="sd">            module: nn.Module to be checked</span>
+<span class="sd">            force_functorch: If set to ``True``, will use functorch to compute</span>
+<span class="sd">                all per sample gradients. Otherwise, functorch will be used only</span>
+<span class="sd">                for layers without registered grad sampler methods.</span>
+<span class="sd">            use_ghost_clipping: If set to ``True``, Ghost Clipping</span>
+<span class="sd">                will be used for clipping gradients of supported layers. If ``False``, Fast</span>
+<span class="sd">                Gradient Clipping will be used for all layers.</span>
+<span class="sd">        """</span>
+        <span class="k">for</span> <span class="n">m_name</span><span class="p">,</span> <span class="n">m</span> <span class="ow">in</span> <span class="n">trainable_modules</span><span class="p">(</span><span class="n">module</span><span class="p">):</span>
+            <span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">m</span><span class="p">)</span> <span class="ow">in</span> <span class="p">[</span><span class="n">DPRNN</span><span class="p">,</span> <span class="n">DPLSTM</span><span class="p">,</span> <span class="n">DPGRU</span><span class="p">]:</span>
+                <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                    <span class="sa">f</span><span class="s2">"Module name: </span><span class="si">{</span><span class="n">m_name</span><span class="si">}</span><span class="s2">, module type: </span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">m</span><span class="p">)</span><span class="si">}</span><span class="s2">. No hook or functorch is added."</span>
+                <span class="p">)</span>
+
+            <span class="k">elif</span> <span class="n">use_ghost_clipping</span> <span class="ow">and</span> <span class="nb">type</span><span class="p">(</span><span class="n">m</span><span class="p">)</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">NORM_SAMPLERS</span><span class="p">:</span>
+                <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                    <span class="sa">f</span><span class="s2">"Module name: </span><span class="si">{</span><span class="n">m_name</span><span class="si">}</span><span class="s2">, module type: </span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">m</span><span class="p">)</span><span class="si">}</span><span class="s2">, under Ghost Clipping."</span>
+                <span class="p">)</span>
+
+            <span class="k">else</span><span class="p">:</span>
+                <span class="k">if</span> <span class="ow">not</span> <span class="n">force_functorch</span> <span class="ow">and</span> <span class="nb">type</span><span class="p">(</span><span class="n">m</span><span class="p">)</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">GRAD_SAMPLERS</span><span class="p">:</span>
+                    <span class="c1"># When functorch is not enforced, use FGC (hook mode) if the layer has a registered grad_sampler (supported). Otherwise, use FGC (functorch mode).</span>
+                    <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                        <span class="sa">f</span><span class="s2">"Module name: </span><span class="si">{</span><span class="n">m_name</span><span class="si">}</span><span class="s2">, module type: </span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">m</span><span class="p">)</span><span class="si">}</span><span class="s2">, under Fast Gradient Clipping (hook mode)."</span>
+                    <span class="p">)</span>
+                <span class="k">else</span><span class="p">:</span>
+                    <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                        <span class="sa">f</span><span class="s2">"Module name: </span><span class="si">{</span><span class="n">m_name</span><span class="si">}</span><span class="s2">, module type: </span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">m</span><span class="p">)</span><span class="si">}</span><span class="s2">, under Fast Gradient Clipping (functorch mode)."</span>
+                    <span class="p">)</span></div>
+
+
     <span class="nd">@property</span>
     <span class="k">def</span> <span class="nf">per_sample_gradient_norms</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">:</span>
 <span class="w">        </span><span class="sd">"""Returns per sample gradient norms. Note that these are not privatized and should only be used for debugging purposes or in non-private settings"""</span>

api/genindex.html

@@ -295,6 +295,10 @@ <h2 id="L">L</h2>
 </li>
 </ul></li>
 </ul></td>
+<td style="width: 33%; vertical-align: top;"><ul>
+<li><a href="grad_sample_module_fast_gradient_clipping.html#opacus.grad_sample.grad_sample_module_fast_gradient_clipping.GradSampleModuleFastGradientClipping.log_module_gradient_sample_mode">log_module_gradient_sample_mode() (opacus.grad_sample.grad_sample_module_fast_gradient_clipping.GradSampleModuleFastGradientClipping method)</a>
+</li>
+</ul></td>
 </tr></table>
 <h2 id="M">M</h2>
 <table class="indextable genindextable" style="width: 100%"><tr>