tokenize-dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tokenizers\n",
    "import os\n",
    "import datasets\n",
    "import torch\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir('multilingual_tinystories')\n",
    "tokenizer = tokenizers.ByteLevelBPETokenizer(\n",
    "    \"./tiny-stories-spanish-bpe-vocab.json\", \n",
    "    \"./tiny-stories-spanish-bpe-merges.txt\"\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "da2cca6297b145248023d128b395cc8c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Resolving data files:   0%|          | 0/54 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "spanish_tinystories_ds = datasets.load_dataset(\"robrenaud/multilingual_tinystories\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "stories = spanish_tinystories_ds['train']['story']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 345,  337,  393,  318,  323,  294,  718,   13,  375,  388, 1120, 1354,\n",
       "          13,  375,  394, 4316,   13,  748,  319,  410,  318,  323,  294,  718,\n",
       "         667,   13,  375,  388, 4316,   13,  301,  198,  270,  337,  406,  461,\n",
       "        4316,  519,  319,   13,  375,  394,  461, 4316,   13,  284,  337,  302,\n",
       "          25,  329, 2046,  315,  284,  319,  302,   25,  285,  400,  395,  284,\n",
       "         337,  318,  416,   13,  198,  198,  270,  337,  367,  393,  306,  823,\n",
       "          13,  375, 1317,  344,  319,   13,  375,  394,  461, 4316,   13,  284,\n",
       "         319,  311,  966,   13,  375,  311, 3908,   13,  198,  198,  270,  319,\n",
       "         915,   13,  375,  352,  388, 4316,   13,  284,  337,  699,  461, 4316,\n",
       "         306,  311,  415,   13,  375,  318,  399,   13,  284,  319,  318,  416,\n",
       "          13,  198,  198,  584,  519,  319,  406,  351,  294,  319,  318,  416,\n",
       "          13,  408,  406,  351,  352,  388, 4316,   13,  408,  318,  416,  667,\n",
       "          13,  284,  337,  367,  981,   13,  301], dtype=torch.int16)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.tensor(tokenizer.encode(stories[len(stories)//10]).ids, dtype=torch.short)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/5304143 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5304143/5304143 [1:02:18<00:00, 1418.64it/s]\n"
     ]
    }
   ],
   "source": [
    "output_buf = []\n",
    "num_outputs = 0\n",
    "for story in tqdm(stories):\n",
    "    encoded = torch.tensor(tokenizer.encode(story).ids, dtype=torch.short)\n",
    "    output_buf.append(encoded)\n",
    "    if len(output_buf) > 500_000:\n",
    "        torch.save(output_buf, f'tokenized/tokenized-{num_outputs}.pt')\n",
    "        num_outputs += 1\n",
    "        output_buf = []\n",
    "if output_buf:\n",
    "    torch.save(output_buf, f'tokenized/tokenized-{num_outputs}.pt')\n",
    "    num_outputs += 1\n",
    "    output_buf = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mkdir: cannot create directory ‘tokenized’: File exists\n"
     ]
    }
   ],
   "source": [
    "!mkdir tokenized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-rwxr--r-- 1 rrenaud rrenaud 3.6G Jul  2 00:23 train.txt\n"
     ]
    }
   ],
   "source": [
    "!ls -lh tokenized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "cloudspace",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}