Merge pull request #18 from A-Baji/dev

rework cli, fix bug, change download logic

Author: A-Baji

README.md

@@ -11,20 +11,26 @@ DiscordAI Modelizer is primarily used as a subcomponent of [DiscordAI](https://g
 3. The source may now be deleted
 
 ## Commands
-### `discordai_modelizer openai create`
-This command will download the specified chat logs, parse them into a usable dataset, then create a customized model using openai.
-### `discordai_modelizer openai list_jobs`
-This command will list all of the openai customization jobs associated with a specified api key.
-### `discordai_modelizer openai list_models`
-This command will list all of the openai customized models associated with a specified api key.
-### `discordai_modelizer openai follow`
-This command will output the event stream of a specified customization job process.
-### `discordai_modelizer openai status`
-This command will output the status of a specified customization job.
-### `discordai_modelizer openai cancel`
-This command will cancel a specified customization job.
-### `discordai_modelizer openai delete`
-This command will delete a specified customized model.
+### Model
+Commands related to your openAI models
+#### `discordai_modelizer model list`
+List your openAi customized models
+#### `discordai_modelizer model create`
+Create a new openAI customized model by downloading the specified chat logs, parsing them into a usable dataset, and then training a customized model using openai
+
+For a proper usage, see the [guide](https://github.com/A-Baji/discordAI#create-a-new-customized-openai-model) for DiscordAI.
+#### `discordai_modelizer model delete`
+Delete an openAI customized model
+### Job
+Commands related to your openAI jobs
+#### `discordai_modelizer job list`
+List your openAI customization jobs
+#### `discordai_modelizer job follow`
+Follow an openAI customization job
+#### `discordai_modelizer job status`
+Get an openAI customization job's status
+#### `discordai_modelizer job cancel`
+Cancel an openAI customization job
 
 ## Disclaimer
 This application allows users to download the chat history of any channel for which they have permission to invite a bot, and then use those logs to create an openai model based on a user's chat messages. It is important to note that this application should only be used with the consent of all members of the channel. Using this application for malicious purposes, such as impersonation, or without the consent of all members is strictly prohibited.

discordai_modelizer/command_line.py

@@ -12,205 +12,208 @@ def discordai_modelizer():
     )
     command = parser.add_subparsers(dest="command")
 
-    openai = command.add_parser("openai", description="OpenAI model customizer and API wrapper")
-    openai_subcommand = openai.add_subparsers(dest="subcommand")
+    model = command.add_parser("model", description="Commands related to your openAI models")
+    job = command.add_parser("job", description="Commands related to your openAI jobs")
 
-    openai_create = openai_subcommand.add_parser(
-        "create", description="Create a new custom openAI model"
+    model_subcommand = model.add_subparsers(dest="subcommand")
+    job_subcommand = job.add_subparsers(dest="subcommand")
+
+    model_list = model_subcommand.add_parser(
+        "list", description="List your openAi customized models"
+    )
+    model_list.add_argument(
+        "-o", "--openai-key",
+        type=str,
+        dest='openai_key',
+        help="The openAI API key to list the models for",
+    )
+    model_list.add_argument(
+        "--simple",
+        action='store_true',
+        required=False,
+        dest='simple',
+        help="Simplify the output to just the model name, job id, and status",
     )
-    openai_create_required_named = openai_create.add_argument_group(
+
+    model_create = model_subcommand.add_parser(
+        "create",
+        description="Create a new openAI customized model by downloading the specified chat logs, parsing them into a usable dataset, and then training a customized model using openai")
+    model_create_required_named = model_create.add_argument_group(
         "required named arguments"
     )
-    openai_create_required_named.add_argument(
-        "-d", "--discord_token",
+    model_create_required_named.add_argument(
+        "-d", "--discord-token",
         type=str,
         dest='discord_token',
         help="The discord token for your bot",
     )
-    openai_create_required_named.add_argument(
-        "-o", "--openai_key",
+    model_create_required_named.add_argument(
+        "-o", "--openai-key",
         type=str,
         dest='openai_key',
-        help="Your openAI API key",
+        help="The openAI API key to use to create the model",
     )
-    openai_create_required_named.add_argument(
+    model_create_required_named.add_argument(
         "-c", "--channel",
         type=str,
         dest='channel',
         help="The ID of the discord channel you want to use",
     )
-    openai_create_required_named.add_argument(
+    model_create_required_named.add_argument(
         "-u", "--user",
         type=str,
         dest='user',
-        help="The name#ID of the discord user you want to use",
+        help="The username#ID of the discord user you want to use",
     )
-
-    openai_create_optional_named = openai_create.add_argument_group("optional named arguments")
-    openai_create_optional_named.add_argument(
-        "-b", "--base_model",
+    model_create_optional_named = model_create.add_argument_group("optional named arguments")
+    model_create_optional_named.add_argument(
+        "-b", "--base-model",
         choices=["davinci", "curie", "babbage", "ada", "none"],
         default="none",
         required=False,
         dest='base_model',
         help="The base model to use for customization. If none, then skips training step: DEFAULT=none",
     )
-    openai_create_optional_named.add_argument(
-        "-t", "--thought_time",
+    model_create_optional_named.add_argument(
+        "-t", "--thought-time",
         type=int,
         default=10,
         required=False,
         dest='thought_time',
         help="The max amount of time in seconds to consider two individual messages to be part of the same \"thought\": DEFAULT=10",
     )
-    openai_create_optional_named.add_argument(
-        "-m", "--max_entries",
+    model_create_optional_named.add_argument(
+        "-m", "--max-entries",
         type=int,
         default=1000,
         required=False,
         dest='max_entries',
         help="The max amount of entries that may exist in the dataset: DEFAULT=1000",
     )
-    openai_create_optional_named.add_argument(
-        "-r", "--reduce_mode",
+    model_create_optional_named.add_argument(
+        "-r", "--reduce-mode",
         choices=["first", "last", "middle", "even"],
         default="even",
         required=False,
         dest='reduce_mode',
         help="The method to reduce the entry count of the dataset: DEFAULT=even",
     )
-    openai_create_optional_named.add_argument(
-        "--clean",
-        action='store_true',
+    model_create_optional_named.add_argument(
+        "--dirty",
+        action='store_false',
         required=False,
-        dest='clean',
-        help="A flag that can be set to clean up outputted files: DEFAULT=False",
+        dest='dirty',
+        help="Skip the clean up step for outputted files",
     )
-    openai_create_optional_named.add_argument(
+    model_create_optional_named.add_argument(
         "--redownload",
         action='store_true',
         required=False,
         dest='redownload',
-        help="A flag that can be set to redownload the discord chat logs: DEFAULT=False",
+        help="Redownload the discord chat logs",
     )
 
-    openai_list_jobs = openai_subcommand.add_parser(
-        "list_jobs", description="Get a list of your openAI jobs"
+    model_delete = model_subcommand.add_parser(
+        "delete", description="Delete an openAI customized model"
     )
-    openai_list_jobs.add_argument(
-        "-o", "--openai_key",
+    model_delete.add_argument(
+        "-o", "--openai-key",
         type=str,
         dest='openai_key',
-        help="Your openAI API key",
+        help="The openAI API key associated with the model to delete",
     )
-    openai_list_jobs.add_argument(
-        "--simple",
-        action='store_true',
-        required=False,
-        dest='simple',
-        help="Simplify the output to just the model name, job id, and status",
+    model_delete.add_argument(
+        "-m", "--model-id",
+        type=str,
+        dest='model_id',
+        help="Target model id",
     )
 
-    openai_list_models = openai_subcommand.add_parser(
-        "list_models", description="Get a list of your openAI models"
+    job_list = job_subcommand.add_parser(
+        "list", description="List your openAI customization jobs"
     )
-    openai_list_models.add_argument(
-        "-o", "--openai_key",
+    job_list.add_argument(
+        "-o", "--openai-key",
         type=str,
         dest='openai_key',
-        help="Your openAI API key",
+        help="The openAI API key to list the jobs for",
     )
-    openai_list_models.add_argument(
+    job_list.add_argument(
         "--simple",
         action='store_true',
         required=False,
         dest='simple',
         help="Simplify the output to just the model name, job id, and status",
     )
 
-    openai_follow = openai_subcommand.add_parser(
-        "follow", description="Follow a custom openAI model job"
+    job_follow = job_subcommand.add_parser(
+        "follow", description="Follow an openAI customization job"
     )
-    openai_follow.add_argument(
-        "-o", "--openai_key",
+    job_follow.add_argument(
+        "-o", "--openai-key",
         type=str,
         dest='openai_key',
-        help="Your openAI API key",
+        help="The openAI API key associated with the job to follow",
     )
-    openai_follow.add_argument(
-        "-j", "--job_id",
+    job_follow.add_argument(
+        "-j", "--job-id",
         type=str,
         dest='job_id',
         help="Target job id",
     )
 
-    openai_status = openai_subcommand.add_parser(
-        "status", description="Get the status of a custom openAI model job"
+    job_status = job_subcommand.add_parser(
+        "status", description="Get an openAI customization job's status"
     )
-    openai_status.add_argument(
-        "-o", "--openai_key",
+    job_status.add_argument(
+        "-o", "--openai-key",
         type=str,
         dest='openai_key',
-        help="Your openAI API key",
+        help="The openAI API key associated with the job to see the status for",
     )
-    openai_status.add_argument(
-        "-j", "--job_id",
+    job_status.add_argument(
+        "-j", "--job-id",
         type=str,
         dest='job_id',
         help="Target job id",
     )
 
-    openai_cancel = openai_subcommand.add_parser(
-        "cancel", description="Cancel a custom openAI model job"
+    job_cancel = job_subcommand.add_parser(
+        "cancel", description="Cancel an openAI customization job"
     )
-    openai_cancel.add_argument(
-        "-o", "--openai_key",
+    job_cancel.add_argument(
+        "-o", "--openai-key",
         type=str,
         dest='openai_key',
-        help="Your openAI API key",
+        help="The openAI API key associated with the job to cancel",
     )
-    openai_cancel.add_argument(
-        "-j", "--job_id",
+    job_cancel.add_argument(
+        "-j", "--job-id",
         type=str,
         dest='job_id',
         help="Target job id",
     )
 
-    openai_delete = openai_subcommand.add_parser(
-        "delete", description="Delete a custom openAI model"
-    )
-    openai_delete.add_argument(
-        "-o", "--openai_key",
-        type=str,
-        dest='openai_key',
-        help="Your openAI API key",
-    )
-    openai_delete.add_argument(
-        "-m", "--model_id",
-        type=str,
-        dest='model_id',
-        help="Target model id",
-    )
-
     args = parser.parse_args()
-    if args.command == "openai":
+    if args.command == "model":
+        if args.subcommand == "list":
+            openai_wrapper.list_models(args.openai_key, args.simple)
         if args.subcommand == "create":
             customize.create_model(args.discord_token, args.openai_key, args.channel, args.user,
                                    thought_time=args.thought_time, max_entry_count=args.max_entries,
-                                   reduce_mode=args.reduce_mode, base_model=args.base_model, clean=args.clean,
+                                   reduce_mode=args.reduce_mode, base_model=args.base_model, clean=args.dirty,
                                    redownload=args.redownload)
-        if args.subcommand == "list_jobs":
+        if args.subcommand == "delete":
+            openai_wrapper.delete_model(args.openai_key, args.model_id)
+    elif args.command == "job":
+        if args.subcommand == "list":
             openai_wrapper.list_jobs(args.openai_key, args.simple)
-        if args.subcommand == "list_models":
-            openai_wrapper.list_models(args.openai_key, args.simple)
         if args.subcommand == "follow":
             openai_wrapper.follow_job(args.openai_key, args.job_id)
         if args.subcommand == "status":
             openai_wrapper.get_status(args.openai_key, args.job_id)
         if args.subcommand == "cancel":
             openai_wrapper.cancel_job(args.openai_key, args.job_id)
-        if args.subcommand == "delete":
-            openai_wrapper.delete_model(args.openai_key, args.model_id)
 
 
 if __name__ == "__main__":

discordai_modelizer/customize.py

@@ -13,7 +13,7 @@ def create_model(bot_token: str, openai_key: str, channel_id: str, user_id: str,
     files_path = appdirs.user_data_dir(appauthor="Adib Baji", appname="discordai")
 
     # Download logs
-    if not os.path.isfile(f"{files_path}/{channel_user}_logs.json") or redownload:
+    if not os.path.isfile(f"{files_path}/{channel_id}_logs.json") or redownload:
         print("INFO: Exporting chat logs using DiscordChatExporter...")
         print("INFO: This may take a few minutes to hours depending on the message count of the channel")
         print("INFO: Progress will NOT be saved if cancelled")
@@ -26,19 +26,18 @@ def create_model(bot_token: str, openai_key: str, channel_id: str, user_id: str,
             "export",
             "-c", channel_id,
             "-t", bot_token,
-            "-o", f"{channel_user}_logs.json",
-            "-f", "Json",
-            "--filter", f"from:'{user_id}'"
+            "-o", f"{channel_id}_logs.json",
+            "-f", "Json"
         ])
         print("--------------------------DiscordChatExporter---------------------------")
-        shutil.move(f"{channel_user}_logs.json", f"{files_path}/{channel_user}_logs.json")
-        print(f"INFO: Logs saved to {files_path}/{channel_user}_logs.json")
+        shutil.move(f"{channel_id}_logs.json", f"{files_path}/{channel_id}_logs.json")
+        print(f"INFO: Logs saved to {files_path}/{channel_id}_logs.json")
     else:
-        print(f"INFO: Chat logs detected locally at {files_path}/{channel_user}_logs.json... Skipping download.")
+        print(f"INFO: Chat logs detected locally at {files_path}/{channel_id}_logs.json... Skipping download.")
 
     # Parse logs
     print("INFO: Parsing chat logs into a openAI compatible dataset...")
-    parse_logs(f"{files_path}/{channel_user}_logs.json", user_id, thought_time)
+    parse_logs(f"{files_path}/{channel_id}_logs.json", user_id, thought_time)
 
     # Prepare and reduce dataset
     print("INFO: Cleaning up generated dataset...")