Merge pull request #41 from A-Baji/dev

docker + manual data tweaking

Author: A-Baji

Dockerfile

@@ -0,0 +1,8 @@
+FROM python:3.11-alpine
+WORKDIR /main
+RUN apk update
+RUN apk add git
+COPY ./requirements.txt ./setup.py ./README.md /main/
+COPY ./discordai_modelizer /main/discordai_modelizer
+RUN pip3 install --upgrade pip
+RUN pip3 install --no-cache-dir . && rm -R /main/*

discordai_modelizer/command_line.py

@@ -128,6 +128,13 @@ def discordai_modelizer():
         dest='redownload',
         help="Redownload the discord chat logs",
     )
+    model_create_optional_named.add_argument(
+        "--use_existing",
+        action='store_true',
+        required=False,
+        dest='use_existing',
+        help="Use an existing dataset that may have been manually revised",
+    ) 
 
     model_delete = model_subcommand.add_parser(
         "delete", description="Delete an openAI customized model"
@@ -225,7 +232,7 @@ def discordai_modelizer():
             customize.create_model(args.discord_token, args.openai_key, args.channel, args.user,
                                    thought_time=args.thought_time, thought_max=args.thought_max, thought_min=args.thought_min, 
                                    max_entry_count=args.max_entries, reduce_mode=args.reduce_mode, base_model=args.base_model, 
-                                   clean=args.dirty, redownload=args.redownload)
+                                   clean=args.dirty, redownload=args.redownload, use_existing=args.use_existing)
         if args.subcommand == "delete":
             openai_wrapper.delete_model(args.openai_key, args.model_id)
     elif args.command == "job":

discordai_modelizer/customize.py

@@ -9,15 +9,19 @@
 
 
 def create_model(bot_token: str, openai_key: str, channel_id: str, user_id: str, thought_time=10, thought_max: int = None, thought_min = 4,
-                 max_entry_count=1000, reduce_mode="even", base_model="none", clean=False, redownload=False):
-    os.environ["OPENAI_API_KEY"] = openai_key
+                 max_entry_count=1000, reduce_mode="even", base_model="none", clean=False, redownload=False, use_existing=False):
+    os.environ["OPENAI_API_KEY"] = openai_key or os.environ["OPENAI_API_KEY"]
     channel_user = f"{channel_id}_{user_id}"
     files_path = pathlib.Path(appdirs.user_data_dir(appname="discordai"))
     full_logs_path = files_path / f"{channel_id}_logs.json"
     full_dataset_path = files_path / f"{channel_user}_data_set.jsonl"
 
+    if not os.path.isfile(full_dataset_path) and use_existing:
+        print("ERROR: No existing dataset could be found!")
+        return
+
     # Download logs
-    if not os.path.isfile(full_logs_path) or redownload:
+    if (not os.path.isfile(full_logs_path) or redownload) and not use_existing:
         print("INFO: Exporting chat logs using DiscordChatExporter...")
         print("INFO: This may take a few minutes to hours depending on the message count of the channel")
         print("INFO: Progress will NOT be saved if cancelled")
@@ -34,13 +38,18 @@ def create_model(bot_token: str, openai_key: str, channel_id: str, user_id: str,
         print("--------------------------DiscordChatExporter---------------------------")
         shutil.move(f"{channel_id}_logs.json", full_logs_path)
         print(f"INFO: Logs saved to {full_logs_path}")
-    else:
+    elif not use_existing:
         print(f"INFO: Chat logs detected locally at {full_logs_path}... Skipping download.")
 
     # Parse logs
-    print("INFO: Parsing chat logs into an openAI compatible dataset...")
-    parse_logs(full_logs_path, channel_id, user_id, thought_time, thought_max, thought_min)
-    get_lines(full_dataset_path, max_entry_count, reduce_mode)
+    if use_existing:
+        print("INFO: Using existing dataset... Skipping download and parsing.")
+    else:
+        print("INFO: Parsing chat logs into an openAI compatible dataset...")
+        parse_logs(full_logs_path, channel_id, user_id, thought_time, thought_max, thought_min)
+        get_lines(full_dataset_path, max_entry_count, reduce_mode)
+        if not clean:
+            print(f"INFO: Dataset saved to {full_dataset_path}")
 
     # Train customized openAI model
     if base_model in ["davinci", "curie", "babbage", "ada"]:
@@ -59,7 +68,7 @@ def create_model(bot_token: str, openai_key: str, channel_id: str, user_id: str,
         print("INFO: No base model selected... Skipping training.")
 
     # Clean up generated files
-    if clean:
+    if clean and not use_existing:
         try:
             os.remove(full_dataset_path)
         except FileNotFoundError:

discordai_modelizer/openai.py

@@ -33,7 +33,7 @@ def list_models(openai_key: str, simple=False):
 
 
 def follow_job(openai_key: str, job_id: str):
-    os.environ["OPENAI_API_KEY"] = openai_key
+    os.environ["OPENAI_API_KEY"] = openai_key or os.environ["OPENAI_API_KEY"]
     try:
         subprocess.run([
             "openai", "api", "fine_tunes.follow",
@@ -56,5 +56,9 @@ def cancel_job(openai_key: str, job_id: str):
 
 
 def delete_model(openai_key: str, model_name: str):
-    os.environ["OPENAI_API_KEY"] = openai_key
+    confirm = input("Are you sure you want to delete this model? This action is not reversable. Y/N: ")
+    if confirm not in ["Y", "y", "yes", "Yes", "YES"]:
+        print("Cancelling model deletion...")
+        return
+    os.environ["OPENAI_API_KEY"] = openai_key or os.environ["OPENAI_API_KEY"]
     print(openai.Model.delete(model_name))

docker-compose.yaml

@@ -0,0 +1,30 @@
+# VERSION=$(cat discordai_modelizer/version.py | grep -oP '\d+\.\d+\.\d+') docker compose up --build
+# discordai_modelizer model list --simple
+# discordai_modelizer model create -d $DISCORD_TOKEN -c $CHANNEL_ID -u "$USERNAME"
+# discordai_modelizer model delete -m "text-babbage:001"
+# discordai_modelizer job list --simple
+# discordai_modelizer job follow -j ft-V31oOgRGZaVZJvZNQFSvSRBl
+# discordai_modelizer job status -j ft-V31oOgRGZaVZJvZNQFSvSRBl --events
+# discordai_modelizer job cancel -j ft-V31oOgRGZaVZJvZNQFSvSRBl
+version: '2.4'
+services:
+  app:
+    build: .
+    image: discord-ai/discordai_modelizer:${VERSION}
+    privileged: true
+    network_mode: host
+    working_dir: /main
+    environment:
+      - DISCORD_TOKEN
+      - OPENAI_API_KEY
+      - CHANNEL_ID
+      - USERNAME
+      - PYTHONUNBUFFERED=1
+    env_file: ./.env
+    volumes:
+      - ./discordai_modelizer:/usr/local/lib/python3.11/site-packages/discordai_modelizer
+    command:
+      - sh
+      - -c
+      - |
+        tail -f /dev/null