diff --git a/.gitignore b/.gitignore index 1607d05..dc55a9c 100755 --- a/.gitignore +++ b/.gitignore @@ -265,5 +265,4 @@ fabric.properties *.csv .DS_Store .ruff_cache/ - -*.json +final_consensus.json diff --git a/src/data/final_consensus.json b/src/data/final_consensus.json deleted file mode 100644 index 924144e..0000000 --- a/src/data/final_consensus.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "initial_conversation": [ - { - "role": "system", - "content": "You are an expert mathematician. For every math problem, provide a detailed, step-by-step explanation of your reasoning and calculations, clearly showing all work and intermediate steps. Ensure that your final answer is clearly stated and justified." - }, - { - "role": "user", - "content": "Tom tells me that one of his two children is a girl. What is the probability that his other child is also a girl?" - } - ], - "iteration_0": { - "qwen/qwen-vl-plus:free": "To solve this problem, we need to consider the possible combinations of children Tom could have, given that he has at least one daughter. Since Tom has two children, the possible combinations are:\n\n1. Boy (B) and Girl (G)\n2. Girl (G) and Girl (G)\n3. Girl (G) and Boy (B)\n\nGiven that Tom tells us he has at least one daughter, we can eliminate the first combination (Boy and Girl) because it does not meet the condition of having at least one daughter. This leaves us with two possible combinations:\n\n1. Girl and Girl\n2. Girl and Boy\n\nSince these two combinations are equally likely, the probability that Tom's other child is also a girl, given that he has at least one daughter, is 1 out of 2, or 50%.", - "sophosympatheia/rogue-rose-103b-v0.2:free": " Let's call Tom's children child A and child B. We know that one of them is a girl, but we don't know which one. We can use a probability tree diagram to visualize the possible scenarios and calculate the probability.\n\nThere are four possible scenarios for the gender of Tom's children:\n\n1. AA (both children are girls)\n2. AG (child A is a girl and child B is a boy)\n3. GA (child A is a boy and child B is a girl)\n4. BB (both children are boys)\n\nWe are given that one of the children is a girl, so we can eliminate the BB scenario. Now, we need to find the probability of the other child being a girl given that we know one of them is a girl.\n\nUsing Bayes' theorem:\n\nP(B|A) = P(A and B) / P(A)\n", - "meta-llama/llama-3.2-3b-instruct:free": "To solve this problem, we need to consider the possible combinations of children Tom can have. Since one of his children is a girl, we have two possible scenarios:\n\nScenario 1: The first child is a girl, and the second child is a boy.\nScenario 2: The first child is a boy, and the second child is a girl.\n\nSince we know that one of the children is a girl, we can focus on the second scenario, where the first child is a boy. In this case, the probability that the second child is also a girl is 1 out of 2, or 1/2.\n\nHowever, we can also calculate the probability of the first child being a girl and the second child being a girl. In this case, the probability is 1 out of 4, or 1/4.\n\nTo find the overall probability that the second child is a girl, we need to add the probabilities of the two scenarios:\n\nProbability = (Probability of first" - }, - "aggregate_0": "To solve this problem, let's consider the possible combinations of children Tom could have, given that he has at least one daughter. Since Tom has two children, the possible combinations are:\n\n1. Boy (B) and Girl (G)\n2. Girl (G) and Girl (G)\n3. Girl (G) and Boy (B)\n\nGiven that Tom tells us he has at least one daughter, we can eliminate the first combination (Boy and Girl) because it does not meet the condition of having at least one daughter. This leaves us with two possible combinations:\n\n1. Girl and Girl\n2. Girl and Boy\n\nSince these two combinations are equally likely, the probability that Tom's other child is also a girl, given that he has at least one daughter, is 1 out of 2, or 50%.\n\nHowever, we can also consider the probability of each child's gender independently. Assuming a random distribution of genders, the probability of a child being a girl is ", - "iteration_1": { - "qwen/qwen-vl-plus:free": "Given the information that Tom has at least one daughter, we can eliminate the combination where both children are boys. This leaves us with two possible combinations for Tom's children: Girl and Girl, or Girl and Boy. Since these two scenarios are equally likely, the probability that Tom's other child is also a girl, given that he has at least one daughter, is 1 out of 2, or 50%.", - "sophosympatheia/rogue-rose-103b-v0.2:free": " Based on the aggregated responses, there are two possible approaches to solve this problem. I will provide both solutions, and you can choose the one that makes more sense to you.\n\nApproach 1 (Conditional Probability):\n\nLet G be the event that the other child is a girl, and D be the event that one of the children is a girl (given). We are looking for P(G|D), the probability that the other child is a girl, given that we know one of the children is a girl.\n\nWe can use Bayes' rule to solve this:\n\nP(G|D) = P(D|G) * P(G) / P(D)\n\nSince we know that one child is a girl, we can simplify the equation:\n\nP(G|D) = P(G) / (1 - P(GG))\n\nGiven that the probability of having", - "meta-llama/llama-3.2-3b-instruct:free": "I see what you're asking now!\n\nLet's re-evaluate the problem. We know that one of Tom's two children is a girl. This means that the probability of the first child being a girl is 1/2.\n\nNow, if the first child is a girl, there are two possibilities for the second child: it could also be a girl, or it could be a boy. Since the two children are independent, the probability of the second child being a girl is also 1/2.\n\nSo, the probability that Tom's other child is also a girl, given that he has at least one daughter, is:\n\nP(other child is girl | at least one daughter) = P(other child is girl) = 1/2\n\nThis makes sense, because if one child is a girl, the probability of the other child being a girl is the same as the probability of a child being a girl in the first place.\n\nTherefore, the probability that Tom's other child is" - }, - "aggregate_1": "Based on the aggregated responses, we can deduce that the probability of Tom's other child being a girl, given that he has at least one daughter, is a topic of discussion among various models. Some models, such as qwen/qwen-vl-plus and sophosympatheia/rogue-rose-103b-v0.2, approach this problem from a conditional probability perspective, while others, like meta-llama/llama-3.2-3b-instruct, take a more straightforward approach by considering the independence of the two children.\n\nFrom the responses, we can see that some models, such as sophosympatheia/rogue-rose-103b-v0.2, propose using Bayes' rule to solve the problem. However, this approach may not be necessary, as the problem can be simplified by considering the independence of the two children. In this case, the probability of the second child being a girl is also 1/2", - "iteration_2": { - "qwen/qwen-vl-plus:free": "To determine the probability that Tom's other child is also a girl, given that he has at least one daughter, we can use a straightforward approach based on the independence of the two children. This problem can be solved without invoking Bayes' rule or conditional probability, as the events are independent.\n\nHere's the step-by-step reasoning:\n\n1. **Identify the possible outcomes for the two children:**\n - The first child can be either a boy (B) or a girl (G).\n - The second child can also be either a boy (B) or a girl (G).\n\n2. **List all possible combinations of the two children:**\n - (Boy, Boy) (B, B)\n - (Boy, Girl) (B, G)\n - (Girl, Boy) (G, B)\n - (Girl, Girl) (G, G)\n\n3. **Identify the outcomes where at least one child is a girl:**\n -", - "sophosympatheia/rogue-rose-103b-v0.2:free": " Based on the aggregated responses, we can conclude that the most appropriate approach to this problem is to assume the independence of the two children's genders. Using this assumption, we can calculate the probability of the other child being a girl as 1/2, since the probability of a child being a girl or a boy is assumed to be the same (ignoring any negligible biological differences).\n\nTo arrive at this answer, we can consider the following reasoning:\n\n1. We are given that Tom has two children, and we know the gender of one of them, which is a girl. This information does not directly affect the gender distribution of the other child, as we are assuming independence between the genders of the two children.\n2. Since we are assuming independence, we can treat the gender of the other child as if it were selected from the same underlying population distribution of 50% girls and 50% boys, which is the same", - "meta-llama/llama-3.2-3b-instruct:free": "The problem can be solved by considering the independence of the two children. Since we only know that one child is a girl, we cannot determine the gender of the other child. Therefore, the probability of the other child being a girl is 1/2, based on the fact that each child has an equal chance of being a boy or a girl.\n\nHere is a step-by-step explanation of the calculation:\n\nLet's assume Tom has two children, C1 and C2. We know that one of them is a girl, say C1. We want to find the probability that the other child, C2, is also a girl.\n\nSince we don't have any information about the gender of C2, we can assume that the probability of C2 being a girl is equal to the probability of C2 being a boy, which is 1/2.\n\nThe probability of the two children being girls is the product of their individual probabilities:\n\nP(C1 is girl) = 1 (" - }, - "aggregate_2": "To determine the probability that Tom's other child is also a girl, given that he has at least one daughter, we can use a straightforward approach based on the independence of the two children. This problem can be solved without invoking Bayes' rule or conditional probability, as the events are independent.\n\nHere's the step-by-step reasoning:\n\n1. **Identify the possible outcomes for the two children:**\n - The first child can be either a boy (B) or a girl (G).\n - The second child can also be either a boy (B) or a girl (G).\n\n2. **List all possible combinations of the two children:**\n - (Boy, Boy) (B, B)\n - (Boy, Girl) (B, G)\n - (Girl, Boy) (G, B)\n - (Girl, Girl) (G, G)\n\n3. **Identify the outcomes where at least one child is a girl:**\n -" -} \ No newline at end of file diff --git a/src/flare_ai_consensus/config.py b/src/flare_ai_consensus/config.py index b43d262..7cac1f2 100644 --- a/src/flare_ai_consensus/config.py +++ b/src/flare_ai_consensus/config.py @@ -39,4 +39,4 @@ class Config(BaseSettings): # Create a global settings instance config = Config() -logger.debug("settings", settings=config.model_dump()) +logger.debug("config", settings=config.model_dump()) diff --git a/tests/working_models.py b/tests/working_models.py index eaf0f31..f10eaf7 100644 --- a/tests/working_models.py +++ b/tests/working_models.py @@ -47,7 +47,7 @@ async def _test_model_completion( "model": model_id, "messages": [{"role": "user", "content": test_prompt}], "max_tokens": model.max_tokens, - "temperature": model.max_tokens, + "temperature": model.temperature, } send_func = client.send_chat_completion else: @@ -86,7 +86,6 @@ async def filter_working_models( Asynchronously tests each model in free_models with the given test prompt and API endpoint returning only those models that respond without an error. - :param client: An instance of AsyncOpenRouterClient. :param free_models: A list of model dictionaries. :param test_prompt: The prompt to test. @@ -97,16 +96,17 @@ async def filter_working_models( _test_model_completion(client, model, test_prompt, api_endpoint, delay=i * 3) for i, model in enumerate(free_models) ] - results = await asyncio.gather(*tasks) + results = await asyncio.gather(*tasks, return_exceptions=True) - valid_models = [] + working_models = [] for result in results: if isinstance(result, Exception): continue - model, works = result + model, works = result # pyright: ignore [reportGeneralTypeIssues] if works: - valid_models.append(model) - return valid_models + working_models.append(model) + + return working_models async def main() -> None: