auto-generating sphinx docs

Author: pytorchbot

main/_modules/torchtune/rlhf/loss/dpo.html

@@ -457,7 +457,6 @@ <h1>Source code for torchtune.rlhf.loss.dpo</h1><div class="highlight"><pre>
 <span class="kn">import</span><span class="w"> </span><span class="nn">torch</span>
 <span class="kn">import</span><span class="w"> </span><span class="nn">torch.nn</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">nn</span>
 <span class="kn">import</span><span class="w"> </span><span class="nn">torch.nn.functional</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">F</span>
-<span class="kn">from</span><span class="w"> </span><span class="nn">torchtune.utils._logging</span><span class="w"> </span><span class="kn">import</span> <span class="n">deprecated</span>
 
 
 <div class="viewcode-block" id="DPOLoss"><a class="viewcode-back" href="../../../../generated/torchtune.rlhf.loss.DPOLoss.html#torchtune.rlhf.loss.DPOLoss">[docs]</a><span class="k">class</span><span class="w"> </span><span class="nc">DPOLoss</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
@@ -607,80 +606,6 @@ <h1>Source code for torchtune.rlhf.loss.dpo</h1><div class="highlight"><pre>
         <span class="p">)</span>
 
         <span class="k">return</span> <span class="n">losses</span><span class="p">,</span> <span class="n">chosen_rewards</span><span class="p">,</span> <span class="n">rejected_rewards</span></div></div>
-
-
-<div class="viewcode-block" id="SimPOLoss"><a class="viewcode-back" href="../../../../generated/torchtune.rlhf.loss.SimPOLoss.html#torchtune.rlhf.loss.SimPOLoss">[docs]</a><span class="nd">@deprecated</span><span class="p">(</span><span class="n">msg</span><span class="o">=</span><span class="s2">&quot;SimPOLoss will be deprecated in an upcoming release.&quot;</span><span class="p">)</span>
-<span class="k">class</span><span class="w"> </span><span class="nc">SimPOLoss</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">    SimPO: Simple Preference Optimization with a Reference-Free Reward: https://arxiv.org/abs/2405.14734.</span>
-<span class="sd">    Intuition from the paper:</span>
-
-<span class="sd">        The effectiveness of SimPO is attributed to a key design: using the average log probability of a sequence as</span>
-<span class="sd">        the implicit reward. Additionally, we introduce a target reward margin to the Bradley-Terry objective to</span>
-<span class="sd">        encourage a larger margin between the winning and losing responses, further enhancing the algorithm&#39;s performance.</span>
-
-<span class="sd">    Based on the TRL implementation:</span>
-<span class="sd">    https://github.com/huggingface/trl/blob/98ad01ddfd1e1b67ec018014b83cba40e0caea66/trl/trainer/cpo_trainer.py#L603</span>
-
-<span class="sd">    SimPO is pretty much identitcal to DPO but uses average logprobs to eliminate the need for a reference model to regularize</span>
-<span class="sd">    the policy during training. It also uses a target reward margin to guide the policy towards better responses.</span>
-<span class="sd">    This is kind of the same intuition as in :class:`~torchtune.rlhf.loss.IPOLoss`, but instead of optimizing against</span>
-<span class="sd">    a margin between the reference policy and policy models, we&#39;re optimizing against a margin between the chosen and</span>
-<span class="sd">    rejected responses.</span>
-
-<span class="sd">    Args:</span>
-<span class="sd">        beta (float): Equivalent temperature scaling parameter to DPO loss, typically in the range of 2.0 to 2.5. Default is 2.0.</span>
-<span class="sd">        gamma (float): Target reward margin hyperparameter, typically we have ``gamma in (0, 1.5]``.</span>
-<span class="sd">            Default is 0.5.</span>
-<span class="sd">        label_smoothing (float): Parameter encoding uncertainty about the labels. Default is 0.</span>
-<span class="sd">    &quot;&quot;&quot;</span>
-
-    <span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span>
-        <span class="bp">self</span><span class="p">,</span>
-        <span class="n">beta</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">2.0</span><span class="p">,</span>
-        <span class="n">gamma</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.5</span><span class="p">,</span>
-        <span class="n">label_smoothing</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.0</span><span class="p">,</span>
-    <span class="p">):</span>
-        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">beta</span> <span class="o">=</span> <span class="n">beta</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span> <span class="o">=</span> <span class="n">gamma</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">label_smoothing</span> <span class="o">=</span> <span class="n">label_smoothing</span>
-
-    <span class="k">def</span><span class="w"> </span><span class="nf">forward</span><span class="p">(</span>
-        <span class="bp">self</span><span class="p">,</span>
-        <span class="n">policy_chosen_logps</span><span class="p">:</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">,</span>
-        <span class="n">policy_rejected_logps</span><span class="p">:</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">,</span>
-    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">]:</span>
-<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">        Compute the SimPO loss for a batch chosen and rejected average log probabilities.</span>
-
-<span class="sd">        Args:</span>
-<span class="sd">            policy_chosen_logps (torch.Tensor): Average log probabilities of the policy model</span>
-<span class="sd">                for the chosen responses with shape [b,].</span>
-<span class="sd">            policy_rejected_logps (torch.Tensor): Average log probabilities of the policy model</span>
-<span class="sd">                for the rejected responses with shape [b,].</span>
-
-<span class="sd">        Returns:</span>
-<span class="sd">            Tuple[torch.Tensor, torch.Tensor, torch.Tensor]; A tuple of three tensors with shape [b,]:</span>
-<span class="sd">                - losses: The SimPO loss for each example in the batch.</span>
-<span class="sd">                - chosen_rewards: Rewards for the chosen responses.</span>
-<span class="sd">                - rejected_rewards: Rewards for the rejected responses.</span>
-<span class="sd">        &quot;&quot;&quot;</span>
-
-        <span class="n">pi_logratios</span> <span class="o">=</span> <span class="n">policy_chosen_logps</span> <span class="o">-</span> <span class="n">policy_rejected_logps</span>
-
-        <span class="n">gamma_logratios</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">beta</span>
-        <span class="n">logits</span> <span class="o">=</span> <span class="n">pi_logratios</span> <span class="o">-</span> <span class="n">gamma_logratios</span>
-
-        <span class="n">losses</span> <span class="o">=</span> <span class="p">(</span>
-            <span class="o">-</span><span class="n">F</span><span class="o">.</span><span class="n">logsigmoid</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">beta</span> <span class="o">*</span> <span class="n">logits</span><span class="p">)</span> <span class="o">*</span> <span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">label_smoothing</span><span class="p">)</span>
-            <span class="o">-</span> <span class="n">F</span><span class="o">.</span><span class="n">logsigmoid</span><span class="p">(</span><span class="o">-</span><span class="bp">self</span><span class="o">.</span><span class="n">beta</span> <span class="o">*</span> <span class="n">logits</span><span class="p">)</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">label_smoothing</span>
-        <span class="p">)</span>
-
-        <span class="n">chosen_rewards</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">beta</span> <span class="o">*</span> <span class="p">(</span><span class="n">policy_chosen_logps</span><span class="p">)</span><span class="o">.</span><span class="n">detach</span><span class="p">()</span>
-        <span class="n">rejected_rewards</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">beta</span> <span class="o">*</span> <span class="p">(</span><span class="n">policy_rejected_logps</span><span class="p">)</span><span class="o">.</span><span class="n">detach</span><span class="p">()</span>
-
-        <span class="k">return</span> <span class="n">losses</span><span class="p">,</span> <span class="n">chosen_rewards</span><span class="p">,</span> <span class="n">rejected_rewards</span></div>
 </pre></div>
 
              </article>

main/_sources/api_ref_rlhf.rst.txt

@@ -16,4 +16,3 @@ Components and losses for RLHF algorithms like PPO and DPO.
     loss.PPOLoss
     loss.DPOLoss
     loss.RSOLoss
-    loss.SimPOLoss

main/_sources/generated/torchtune.rlhf.loss.SimPOLoss.rst.txt

@@ -56,8 +56,6 @@ To use any of these, simply use the ``loss`` config entry or flag through the :r
     loss=torchtune.modules.loss.RSOLoss \
     gamma=0.5
 
-.. todo (@SalmanMohammadi) point to an example repo for SimPO
-
 For a deeper understanding of the different levers you can pull when using this recipe,
 see our documentation for the different PEFT training paradigms we support:
 

main/_sources/recipes/dpo.rst.txt

@@ -473,9 +473,6 @@ <h1>torchtune.rlhf<a class="headerlink" href="#torchtune-rlhf" title="Permalink
 <tr class="row-even"><td><p><a class="reference internal" href="generated/torchtune.rlhf.loss.RSOLoss.html#torchtune.rlhf.loss.RSOLoss" title="torchtune.rlhf.loss.RSOLoss"><code class="xref py py-obj docutils literal notranslate"><span class="pre">loss.RSOLoss</span></code></a></p></td>
 <td><p>Statistical Rejection Sampling Optimization (RSO) or &quot;hinge&quot; loss module: <a class="reference external" href="https://arxiv.org/abs/2309.06657">https://arxiv.org/abs/2309.06657</a>.</p></td>
 </tr>
-<tr class="row-odd"><td><p><a class="reference internal" href="generated/torchtune.rlhf.loss.SimPOLoss.html#torchtune.rlhf.loss.SimPOLoss" title="torchtune.rlhf.loss.SimPOLoss"><code class="xref py py-obj docutils literal notranslate"><span class="pre">loss.SimPOLoss</span></code></a></p></td>
-<td><p>Simple Preference Optimization with a Reference-Free Reward: <a class="reference external" href="https://arxiv.org/abs/2405.14734">https://arxiv.org/abs/2405.14734</a>.</p></td>
-</tr>
 </tbody>
 </table>
 </section>

main/api_ref_rlhf.html

@@ -41,7 +41,7 @@
     <link rel="index" title="Index" href="genindex.html" />
     <link rel="search" title="Search" href="search.html" />
     <link rel="next" title="FullModelHFCheckpointer" href="generated/torchtune.training.FullModelHFCheckpointer.html" />
-    <link rel="prev" title="torchtune.rlhf.loss.SimPOLoss" href="generated/torchtune.rlhf.loss.SimPOLoss.html" />
+    <link rel="prev" title="RSOLoss" href="generated/torchtune.rlhf.loss.RSOLoss.html" />
   <!-- Google Tag Manager -->
     <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
     new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
@@ -620,7 +620,7 @@ <h2>Miscellaneous<a class="headerlink" href="#miscellaneous" title="Permalink to
         <a href="generated/torchtune.training.FullModelHFCheckpointer.html" class="btn btn-neutral float-right" title="FullModelHFCheckpointer" accesskey="n" rel="next">Next <img src="_static/images/chevron-right-orange.svg" class="next-page"></a>
 
 
-        <a href="generated/torchtune.rlhf.loss.SimPOLoss.html" class="btn btn-neutral" title="torchtune.rlhf.loss.SimPOLoss" accesskey="p" rel="prev"><img src="_static/images/chevron-right-orange.svg" class="previous-page"> Previous</a>
+        <a href="generated/torchtune.rlhf.loss.RSOLoss.html" class="btn btn-neutral" title="RSOLoss" accesskey="p" rel="prev"><img src="_static/images/chevron-right-orange.svg" class="previous-page"> Previous</a>
 
     </div>
 

main/api_ref_training.html

@@ -40,7 +40,7 @@
   <link rel="stylesheet" href="../_static/css/custom_torchtune.css" type="text/css" />
     <link rel="index" title="Index" href="../genindex.html" />
     <link rel="search" title="Search" href="../search.html" />
-    <link rel="next" title="torchtune.rlhf.loss.SimPOLoss" href="torchtune.rlhf.loss.SimPOLoss.html" />
+    <link rel="next" title="torchtune.training" href="../api_ref_training.html" />
     <link rel="prev" title="DPOLoss" href="torchtune.rlhf.loss.DPOLoss.html" />
   <!-- Google Tag Manager -->
     <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
@@ -516,7 +516,7 @@ <h1>RSOLoss<a class="headerlink" href="#rsoloss" title="Permalink to this headin
 
     <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
 
-        <a href="torchtune.rlhf.loss.SimPOLoss.html" class="btn btn-neutral float-right" title="torchtune.rlhf.loss.SimPOLoss" accesskey="n" rel="next">Next <img src="../_static/images/chevron-right-orange.svg" class="next-page"></a>
+        <a href="../api_ref_training.html" class="btn btn-neutral float-right" title="torchtune.training" accesskey="n" rel="next">Next <img src="../_static/images/chevron-right-orange.svg" class="next-page"></a>
 
 
         <a href="torchtune.rlhf.loss.DPOLoss.html" class="btn btn-neutral" title="DPOLoss" accesskey="p" rel="prev"><img src="../_static/images/chevron-right-orange.svg" class="previous-page"> Previous</a>