-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathsaved_models.py
More file actions
72 lines (56 loc) · 4 KB
/
saved_models.py
File metadata and controls
72 lines (56 loc) · 4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
from src.models.TopKSaes import MatryoshkaTopKSae, TopKSae
#load gemma-scope-2b-pt-res-canonical
#From: https://github.com/jbloomAus/SAELens/blob/a470460/sae_lens/pretrained_saes.yaml#L1667
GEMMA_SCOPE_VERSIONS = [
"layer_2/width_16k/canonical",
"layer_2/width_65k/canonical"
]
GEMMA_SCOPE_RELEASE_ID = "gemma-scope-2b-pt-res-canonical"
TOPK_16K_PATHS = [
(TopKSae, '/root/pile_uncopyrighted/logs/first-run/20241205_133221/step_12208_tokens_50003968', 'TopK_16k_128'),
(TopKSae,'/root/pile_uncopyrighted/logs/TopKSae_16k_64_20241222_132552/step_12208_tokens_50003968', 'TopK_16k_64'),
(TopKSae, '/root/pile_uncopyrighted/logs/TopKSae_16.4k_topk=32_20241218_102749/step_12208_tokens_50003968', 'TopK_16k_32'),
(TopKSae, '/root/pile_uncopyrighted/logs/first-run/20241206_193501/step_12208_tokens_50003968', 'TopK_16k_16'),
]
TOPK_32K_PATHS = [
(TopKSae, '/root/pile_uncopyrighted/logs/first-run/20241209_203938/step_12208_tokens_50003968', 'TopK_32k_256'),
(TopKSae, '/root/pile_uncopyrighted/logs/first-run/20241205_133228/step_12208_tokens_50003968', 'TopK_32k_128'),
(TopKSae, '/root/pile_uncopyrighted/logs/TopKSae_32.8k_topk=64_20241218_102500/step_12208_tokens_50003968', 'TopK_32k_64'),
(TopKSae, '/root/pile_uncopyrighted/logs/TopKSae_32.8k_topk=32_20241218_110806/step_12208_tokens_50003968', 'TopK_32k_32'),
]
TOPK_65K_PATHS = [
(TopKSae, '/root/pile_uncopyrighted/logs/first-run/20241209_204253/step_12208_tokens_50003968', 'TopK_65k_512'),
(TopKSae, '/root/pile_uncopyrighted/logs/first-run/20241205_133226/step_12208_tokens_50003968', 'TopK_65k_256'),
(TopKSae, '/root/pile_uncopyrighted/logs/TopKSae_65.5k_topk=128_20241218_102811/step_12208_tokens_50003968', 'TopK_65k_128'),
(TopKSae, '/root/pile_uncopyrighted/logs/TopKSae_65.5k_topk=64_20241218_111101/step_12208_tokens_50003968', 'TopK_65k_64'),
]
MATRYOSHKA_SAMPLED_PATHS = [
(MatryoshkaTopKSae, '/root/pile_uncopyrighted/logs/first-run/20241206_194530/step_12208_tokens_50003968', 'MatryoshkaTopk_sampled_65k_256'),
]
MATRYOSHKA_TOPK_SAE_PATHS = [
(MatryoshkaTopKSae, '/root/pile_uncopyrighted/logs/first-run/20241209_203951/step_12208_tokens_50003968', 'MatryoshkaTopk_65k_512'),
(MatryoshkaTopKSae, '/root/pile_uncopyrighted/logs/first-run/20241204_195640/step_12208_tokens_50003968', 'MatryoshkaTopk_65k_256'),
(MatryoshkaTopKSae, '/root/pile_uncopyrighted/logs/MatryoshkaTopKSae_65.5k__linear_3__topks_32_64_128_20241218_102458/step_12208_tokens_50003968', 'MatryoshkaTopk_65k_128'),
(MatryoshkaTopKSae, '/root/pile_uncopyrighted/logs/first-run/20241206_193510/step_12208_tokens_50003968', 'MatryoshkaTopk_65k_64'),
]
TOPK_SAE_PATHS = TOPK_16K_PATHS + TOPK_32K_PATHS + TOPK_65K_PATHS
GEMMA_MODELS = [
(None,
f'/root/pile_uncopyrighted/{path_name.replace("/", "_")}',
name
)
for name, path_name in [('GemmaScope_65k', 'layer_2/width_65k/canonical'), ('GemmaScope_16k', 'layer_2/width_16k/canonical')]
]
ALL_NON_GEMMA_MODELS = TOPK_SAE_PATHS + MATRYOSHKA_TOPK_SAE_PATHS
ALL_MODELS = GEMMA_MODELS + ALL_NON_GEMMA_MODELS
TOPK_65_256 = [
(TopKSae, '/root/pile_uncopyrighted/logs/first-run/20241205_133226/step_12208_tokens_50003968', 'TopK_65k_256')
]
TARGET_MODELS2 = [
#(TopKSae, '/root/pile_uncopyrighted/logs/first-run/20241209_204253/step_12208_tokens_50003968', 'TopK_65k_512'),
#(MatryoshkaTopKSae, '/root/pile_uncopyrighted/logs/first-run/20241209_203951/step_12208_tokens_50003968', 'MatryoshkaTopk_65k_512'),
(MatryoshkaTopKSae, '/root/pile_uncopyrighted/logs/MatryoshkaTopKSae_65.5k__linear_3__topks_32_64_128_20241218_102458/step_12208_tokens_50003968', 'MatryoshkaTopk_65k_128'),
(TopKSae, '/root/pile_uncopyrighted/logs/TopKSae_65.5k_topk=128_20241218_102811/step_12208_tokens_50003968', 'TopK_65k_128'),
#(MatryoshkaTopKSae, '/root/pile_uncopyrighted/logs/first-run/20241206_193510/step_12208_tokens_50003968', 'MatryoshkaTopk_65k_64'),
#(TopKSae, '/root/pile_uncopyrighted/logs/TopKSae_65.5k_topk=64_20241218_111101/step_12208_tokens_50003968', 'TopK_65k_64'),
]