Adding a "Discover" object for finding candidate tables that may be joined on the main table

.gitignore

@@ -20,6 +20,9 @@ var/
 .installed.cfg
 *.egg
 *.pkl
+data/
+
+data/
 
 .pytest_cache/
 

10_discovery.py

@@ -0,0 +1,24 @@
+# %%
+import polars as pl
+
+from skrub._discover import Discover, find_unique_values
+
+# %%
+# working with binary to debug
+data_lake_path = "data/binary_update/*.parquet"
+base_table_path = "data/source_tables/company_employees-yadl-depleted.parquet"
+query_column = "col_to_embed"
+
+
+base_table = pl.read_parquet(base_table_path)
+# %%
+find_unique_values(base_table, ["col_to_embed"])
+# %%
+discover = Discover(data_lake_path, [query_column])
+print("fitting")
+discover.fit(base_table)
+print("transforming")
+ranking = discover.transform(base_table)
+print(ranking)
+
+# %%

CHANGES.rst

@@ -462,6 +462,14 @@ Changes
   :pr:`1236` by :user:`Priscilla Baah<priscilla-b>`
 
 - Optimize the :class:`StringEncoder`: lower memory footprint and faster execution in some cases.
+<<<<<<< HEAD
+  :pr:`1248` by :user:`Gaël Varoquaux <gaelvaroquaux>`
+
+Bug fixes
+---------
+- :class:`StringEncoder` now works correctly in presence of null values.
+=======
+>>>>>>> upstream/main
   :pr:`1248` by :user:`Gaël Varoquaux <gaelvaroquaux>`
 
 Bug fixes

doc/generated_for_index/baskets.html

@@ -0,0 +1,30 @@
+<table border="1" class="dataframe">
+  <thead>
+    <tr style="text-align: right;">
+      <th>basket_ID</th>
+      <th>fraud_flag</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td>85517</td>
+      <td>0</td>
+    </tr>
+    <tr>
+      <td>83008</td>
+      <td>0</td>
+    </tr>
+    <tr>
+      <td>...</td>
+      <td>...</td>
+    </tr>
+    <tr>
+      <td>97639</td>
+      <td>0</td>
+    </tr>
+    <tr>
+      <td>95939</td>
+      <td>0</td>
+    </tr>
+  </tbody>
+</table>

doc/generated_for_index/code_block_0.html

@@ -0,0 +1,12 @@
+<div class="highlight"><pre><span></span><span class="c1"># A dataset with multiple tables</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">skrub</span>
+<span class="kn">from</span><span class="w"> </span><span class="nn">skrub.datasets</span><span class="w"> </span><span class="kn">import</span> <span class="n">fetch_credit_fraud</span>
+<span class="c1"># use the test set to have smaller data</span>
+<span class="n">dataset</span> <span class="o">=</span> <span class="n">fetch_credit_fraud</span><span class="p">(</span><span class="n">split</span><span class="o">=</span><span class="s2">&quot;test&quot;</span><span class="p">)</span>
+
+<span class="c1"># Extract simplified tables</span>
+<span class="c1"># Drop the columns that are not central to the analysis</span>
+<span class="n">products_df</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">products</span><span class="p">[[</span><span class="s2">&quot;basket_ID&quot;</span><span class="p">,</span> <span class="s2">&quot;cash_price&quot;</span><span class="p">,</span> <span class="s2">&quot;Nbr_of_prod_purchas&quot;</span><span class="p">]]</span>
+<span class="c1"># Rename the ID column to &quot;basket_ID&quot;</span>
+<span class="n">baskets_df</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">baskets</span><span class="o">.</span><span class="n">rename</span><span class="p">(</span><span class="n">columns</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;ID&quot;</span><span class="p">:</span> <span class="s2">&quot;basket_ID&quot;</span><span class="p">})</span>
+</pre></div>

doc/generated_for_index/code_block_2.html

@@ -0,0 +1,19 @@
+<div class="highlight"><pre><span></span><span class="c1"># Define the inputs of our skrub pipeline</span>
+<span class="n">products</span> <span class="o">=</span> <span class="n">skrub</span><span class="o">.</span><span class="n">var</span><span class="p">(</span><span class="s2">&quot;products&quot;</span><span class="p">,</span> <span class="n">products_df</span><span class="p">)</span>
+<span class="n">baskets</span> <span class="o">=</span> <span class="n">skrub</span><span class="o">.</span><span class="n">var</span><span class="p">(</span><span class="s2">&quot;baskets&quot;</span><span class="p">,</span> <span class="n">baskets_df</span><span class="p">)</span>
+
+<span class="c1"># Specify our &quot;X&quot; and &quot;y&quot; variables for machine learning</span>
+<span class="n">basket_IDs</span> <span class="o">=</span> <span class="n">baskets</span><span class="p">[[</span><span class="s2">&quot;basket_ID&quot;</span><span class="p">]]</span><span class="o">.</span><span class="n">skb</span><span class="o">.</span><span class="n">mark_as_X</span><span class="p">()</span>
+<span class="n">fraud_flags</span> <span class="o">=</span> <span class="n">baskets</span><span class="p">[</span><span class="s2">&quot;fraud_flag&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">skb</span><span class="o">.</span><span class="n">mark_as_y</span><span class="p">()</span>
+
+<span class="c1"># A pandas-based data-preparation pipeline that merges the tables</span>
+<span class="n">aggregated_products</span> <span class="o">=</span> <span class="n">products</span><span class="o">.</span><span class="n">groupby</span><span class="p">(</span><span class="s2">&quot;basket_ID&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">agg</span><span class="p">(</span>
+    <span class="n">skrub</span><span class="o">.</span><span class="n">choose_from</span><span class="p">((</span><span class="s2">&quot;mean&quot;</span><span class="p">,</span> <span class="s2">&quot;max&quot;</span><span class="p">,</span> <span class="s2">&quot;count&quot;</span><span class="p">)))</span><span class="o">.</span><span class="n">reset_index</span><span class="p">()</span>
+<span class="n">features</span> <span class="o">=</span> <span class="n">basket_IDs</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span><span class="n">aggregated_products</span><span class="p">,</span> <span class="n">on</span><span class="o">=</span><span class="s2">&quot;basket_ID&quot;</span><span class="p">)</span>
+<span class="kn">from</span><span class="w"> </span><span class="nn">sklearn.ensemble</span><span class="w"> </span><span class="kn">import</span> <span class="n">ExtraTreesClassifier</span>
+<span class="n">predictions</span> <span class="o">=</span> <span class="n">features</span><span class="o">.</span><span class="n">skb</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="n">ExtraTreesClassifier</span><span class="p">(),</span> <span class="n">y</span><span class="o">=</span><span class="n">fraud_flags</span><span class="p">)</span>
+
+<span class="c1"># Now use skrub to tune hyperparameters of the above pipeline</span>
+<span class="n">search</span> <span class="o">=</span> <span class="n">predictions</span><span class="o">.</span><span class="n">skb</span><span class="o">.</span><span class="n">make_grid_search</span><span class="p">(</span><span class="n">fitted</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">scoring</span><span class="o">=</span><span class="s2">&quot;roc_auc&quot;</span><span class="p">)</span>
+<span class="n">search</span><span class="o">.</span><span class="n">plot_results</span><span class="p">()</span>
+</pre></div>

doc/generated_for_index/parallel_coordinates.html

@@ -0,0 +1,2 @@
+<div>                        <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-3.3.0.min.js" integrity="sha256-bO3dS6yCpk9aK4gUpNELtCiDeSYvGYnK7jFI58NQnHI=" crossorigin="anonymous"></script>                <div id="935010eb-5f36-4ef3-b0f9-8f9e881f7d93" class="plotly-graph-div" style="height:100%; width:100%;"></div>            <script type="text/javascript">                window.PLOTLYENV=window.PLOTLYENV || {};                                if (document.getElementById("935010eb-5f36-4ef3-b0f9-8f9e881f7d93")) {                    Plotly.newPlot(                        "935010eb-5f36-4ef3-b0f9-8f9e881f7d93",                        [{"dimensions":[{"label":"choose_from(('mean',\u003cbr\u003e\n'max', 'count'))","ticktext":["count","max","mean"],"tickvals":[0,1,2],"values":{"dtype":"f8","bdata":"AAAAAAAAAEB3Jm\u002fNIBPwPwAAAAAAAAAA"}},{"label":"score time","ticktext":["0.0544","0.0550","0.0556","0.0562","0.0568","0.0574","0.0579","0.0585","0.0591","0.0597"],"tickvals":[0.05440521240234375,0.05499526129828559,0.05558531019422743,0.05617535909016927,0.05676540798611111,0.05735545688205295,0.057945505777994795,0.05853555467393663,0.05912560356987848,0.059715652465820314],"values":{"dtype":"f8","bdata":"zczMzAyTrj\u002fChb5EsqGtPwAAAAAA26s\u002f"}},{"label":"fit time","ticktext":["0.431","0.435","0.440","0.444","0.448","0.452","0.456","0.460","0.464","0.468"],"tickvals":[0.4313982963562012,0.4354975117577447,0.4395967271592882,0.4436959425608317,0.44779515796237523,0.45189437336391874,0.45599358876546225,0.46009280416700576,0.4641920195685493,0.4682912349700928],"values":{"dtype":"f8","bdata":"zczMzHv43T8Ay8l75WbdP5qZmZkHnNs\u002f"}},{"label":"score","ticktext":["0.584","0.593","0.602","0.611","0.620","0.628","0.637","0.646","0.655","0.664"],"tickvals":[0.5839160882333971,0.5928275792225978,0.6017390702117984,0.610650561200999,0.6195620521901997,0.6284735431794004,0.637385034168601,0.6462965251578017,0.6552080161470023,0.6641195071362029],"values":{"dtype":"f8","bdata":"4iN5jXdA5T9Yr\u002fRndtnkPxpD0spwr+I\u002f"}}],"labelangle":15,"labelside":"top","line":{"color":{"dtype":"f8","bdata":"4iN5jXdA5T9Yr\u002fRndtnkPxpD0spwr+I\u002f"},"colorbar":{"title":{"text":"score"}},"colorscale":[[0.0,"rgb(0,0,255)"],[1.0,"rgb(255,0,0)"]],"showscale":true},"type":"parcoords"}],                        {"font":{"size":18},"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermap":[{"type":"scattermap","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}}},                        {"responsive": true}                    )                };            </script>        </div>