Skip to content

Commit 4d7d98c

Browse files
zhuwq0zhuwq0
committed
add cctorch to california
1 parent 436cfce commit 4d7d98c

File tree

4 files changed

+265
-34
lines changed

4 files changed

+265
-34
lines changed

‎examples/california/cut_templates_cc.py‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -421,8 +421,15 @@ def cut_templates(jdays, root_path, region, config, bucket, protocol, token):
421421
stations = pd.read_json(fp, orient="index")
422422
stations["station_id"] = stations.index
423423
stations.sort_values(by=["latitude", "longitude"], inplace=True)
424-
print(f"{len(stations) = }")
425-
print(stations.iloc[:5])
424+
425+
# ############### DEBUG ###############
426+
# "minlatitude": 35.205,
427+
# "maxlatitude": 36.205,
428+
# "minlongitude": -118.004,
429+
# "maxlongitude": -117.004,
430+
minlat, maxlat = 35.205, 36.205
431+
minlon, maxlon = -118.004, -117.004
432+
# ############### DEBUG ###############
426433

427434
# %%
428435
# events = pd.read_csv(f"{root_path}/{data_path}/ransac_events.csv", parse_dates=["time"])
@@ -435,6 +442,18 @@ def cut_templates(jdays, root_path, region, config, bucket, protocol, token):
435442
with fs.open(f"{bucket}/{data_path}/{year:04d}/adloc_events_{jday:03d}.csv", "r") as fp:
436443
events = pd.read_csv(fp, parse_dates=["time"])
437444

445+
# ############### DEBUG ###############
446+
# events = events[
447+
# (events["latitude"] >= minlat)
448+
# & (events["latitude"] <= maxlat)
449+
# & (events["longitude"] >= minlon)
450+
# & (events["longitude"] <= maxlon)
451+
# ]
452+
# plt.figure(figsize=(10, 10))
453+
# plt.scatter(events["longitude"], events["latitude"], s=1)
454+
# plt.savefig(f"events_{year:04d}_{jday:03d}.png")
455+
# ############### DEBUG ###############
456+
438457
events.rename(columns={"time": "event_time"}, inplace=True)
439458
events["event_time"] = pd.to_datetime(events["event_time"], utc=True)
440459
reference_t0 = events["event_time"].min()
@@ -467,6 +486,10 @@ def cut_templates(jdays, root_path, region, config, bucket, protocol, token):
467486
with fs.open(f"{bucket}/{data_path}/{year:04d}/adloc_picks_{jday:03d}.csv", "r") as fp:
468487
picks = pd.read_csv(fp)
469488

489+
# ############### DEBUG ###############
490+
# picks = picks[(picks["event_index"].isin(events["event_index"]))]
491+
# ############### DEBUG ###############
492+
470493
picks = picks[picks["adloc_mask"] == 1]
471494
picks["phase_time"] = pd.to_datetime(picks["phase_time"], utc=True)
472495
min_phase_score = picks["phase_score"].min()
@@ -507,6 +530,12 @@ def cut_templates(jdays, root_path, region, config, bucket, protocol, token):
507530
print(f"{len(picks) = }")
508531
picks = picks[picks["dist_km"] < config["max_epicenter_dist_km"]]
509532
print(f"{len(picks) = } with dist_km < {config['max_epicenter_dist_km']} km")
533+
print(picks.iloc[:5])
534+
535+
## filter stations
536+
stations = stations[stations["station_id"].isin(picks["station_id"])]
537+
print(f"{len(stations) = }")
538+
print(stations.iloc[:5])
510539

511540
# %% Reindex event and station to make it continuous
512541
stations["idx_sta"] = np.arange(len(stations))

‎examples/california/run_cctorch.py‎

Lines changed: 44 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,33 @@
11
# %%
2+
import argparse
23
import os
34

45
import torch
5-
from args import parse_args
6+
7+
8+
##
9+
def parse_args():
10+
parser = argparse.ArgumentParser(description="Run Gamma on NCEDC/SCEDC data")
11+
parser.add_argument("--num_nodes", type=int, default=1)
12+
parser.add_argument("--node_rank", type=int, default=0)
13+
parser.add_argument("--year", type=int, default=2023)
14+
parser.add_argument("--root_path", type=str, default="local")
15+
parser.add_argument("--region", type=str, default="Cal")
16+
parser.add_argument("--bucket", type=str, default="quakeflow_catalog")
17+
return parser.parse_args()
18+
619

720
args = parse_args()
21+
22+
##
23+
year = 2019
24+
jday = 185
25+
826
# %%
927
root_path = args.root_path
1028
region = args.region
1129

12-
data_path = f"{region}/cctorch"
30+
data_path = f"{region}/cctorch/{year}"
1331
result_path = f"{region}/cctorch/ccpairs"
1432

1533
# data_path = f"{region}/cctorch_ca"
@@ -28,31 +46,14 @@
2846
block_size1 = 1000_000
2947
block_size2 = 1000_000
3048

31-
if args.dtct_pair:
32-
dt_ct = f"{root_path}/{region}/hypodd/dt.ct"
33-
pair_list = f"{root_path}/{region}/hypodd/pairs.txt"
34-
lines = []
35-
with open(dt_ct, "r") as fp:
36-
for line in fp:
37-
if line.startswith("#"):
38-
ev1, ev2 = line.split()[1:3]
39-
if ev1 > ev2:
40-
ev1, ev2 = ev2, ev1
41-
lines.append(f"{ev1},{ev2}\n")
42-
43-
print(f"Number of pairs from hypodd dt.ct: {len(lines)}")
44-
with open(f"{root_path}/{region}/hypodd/pairs.txt", "w") as fp:
45-
fp.writelines(lines)
46-
base_cmd = f"../CCTorch/run.py --pair_list={root_path}/{region}/hypodd/pairs.txt --data_path1={root_path}/{region}/cctorch/template.dat --data_format1=memmap --config={root_path}/{region}/cctorch/config.json --batch_size={batch} --block_size1={block_size1} --block_size2={block_size2} --result_path={root_path}/{result_path}"
4749

48-
else:
49-
base_cmd = (
50-
f"../CCTorch/run.py --pair_list={root_path}/{data_path}/pairs.txt --data_path1={root_path}/{data_path}/template.dat --data_format1=memmap "
51-
f"--data_list1={root_path}/{data_path}/cctorch_picks.csv "
52-
f"--events_csv={root_path}/{data_path}/cctorch_events.csv --picks_csv={root_path}/{data_path}/cctorch_picks.csv --stations_csv={root_path}/{data_path}/cctorch_stations.csv "
53-
f"--config={root_path}/{data_path}/config.json --batch_size={batch} --block_size1={block_size1} --block_size2={block_size2} "
54-
f"--result_path={root_path}/{result_path}"
55-
)
50+
base_cmd = (
51+
f"../../CCTorch/run.py --pair_list={root_path}/{data_path}/pairs_{jday:03d}.txt --data_path1={root_path}/{data_path}/template_{jday:03d}.dat --data_format1=memmap "
52+
f"--data_list1={root_path}/{data_path}/cctorch_picks_{jday:03d}.csv "
53+
f"--events_csv={root_path}/{data_path}/cctorch_events_{jday:03d}.csv --picks_csv={root_path}/{data_path}/cctorch_picks_{jday:03d}.csv --stations_csv={root_path}/{data_path}/cctorch_stations_{jday:03d}.csv "
54+
f"--config={root_path}/{data_path}/config_{jday:03d}.json --batch_size={batch} --block_size1={block_size1} --block_size2={block_size2} "
55+
f"--result_path={root_path}/{result_path}"
56+
)
5657

5758

5859
if torch.cuda.is_available():
@@ -70,23 +71,34 @@
7071
else:
7172
cmd = f"python {base_cmd} --device={device}"
7273
print(cmd)
73-
os.system(cmd)
74+
# os.system(cmd)
7475

7576
# %%
7677
for rank in range(num_gpu):
7778
if not os.path.exists(f"{root_path}/{result_path}/CC_{rank:03d}_{num_gpu:03d}.csv"):
7879
continue
7980
if rank == 0:
80-
cmd = f"cat {root_path}/{result_path}/CC_{rank:03d}_{num_gpu:03d}.csv > {root_path}/{data_path}/dtcc.csv"
81+
cmd = f"cat {root_path}/{result_path}/CC_{rank:03d}_{num_gpu:03d}.csv > {root_path}/{data_path}/../dtcc.csv"
8182
else:
82-
cmd = (
83-
f"tail -n +2 {root_path}/{result_path}/CC_{rank:03d}_{num_gpu:03d}.csv >> {root_path}/{data_path}/dtcc.csv"
84-
)
83+
cmd = f"tail -n +2 {root_path}/{result_path}/CC_{rank:03d}_{num_gpu:03d}.csv >> {root_path}/{data_path}/../dtcc.csv"
8584
print(cmd)
8685
os.system(cmd)
8786

8887

89-
cmd = f"cat {root_path}/{result_path}/CC_*_{num_gpu:03d}_dt.cc > {root_path}/{data_path}/dt.cc"
88+
cmd = f"cat {root_path}/{result_path}/CC_*_{num_gpu:03d}_dt.cc > {root_path}/{data_path}/../dt.cc"
89+
print(cmd)
90+
os.system(cmd)
91+
92+
##
93+
cmd = f"cp {root_path}/{data_path}/cctorch_stations_{jday:03d}.csv {root_path}/{data_path}/../cctorch_stations.csv"
94+
print(cmd)
95+
os.system(cmd)
96+
97+
cmd = f"cp {root_path}/{data_path}/cctorch_events_{jday:03d}.csv {root_path}/{data_path}/../cctorch_events.csv"
98+
print(cmd)
99+
os.system(cmd)
100+
101+
cmd = f"cp {root_path}/{data_path}/cctorch_picks_{jday:03d}.csv {root_path}/{data_path}/../cctorch_picks.csv"
90102
print(cmd)
91103
os.system(cmd)
92104

‎examples/california/run_hypodd_cc.py‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
# %%
2+
# from args import parse_args
3+
##
4+
import argparse
5+
import json
6+
import os
7+
8+
import numpy as np
9+
import pandas as pd
10+
11+
12+
def parse_args():
13+
parser = argparse.ArgumentParser(description="Run Gamma on NCEDC/SCEDC data")
14+
parser.add_argument("--num_nodes", type=int, default=1)
15+
parser.add_argument("--node_rank", type=int, default=0)
16+
parser.add_argument("--year", type=int, default=2023)
17+
parser.add_argument("--root_path", type=str, default="local")
18+
parser.add_argument("--region", type=str, default="Cal")
19+
parser.add_argument("--bucket", type=str, default="quakeflow_catalog")
20+
return parser.parse_args()
21+
22+
23+
# %%
24+
args = parse_args()
25+
root_path = args.root_path
26+
region = args.region
27+
28+
with open(f"{root_path}/{region}/config.json", "r") as fp:
29+
config = json.load(fp)
30+
31+
# %%
32+
data_path = f"{region}/cctorch"
33+
result_path = f"{region}/hypodd"
34+
if not os.path.exists(f"{root_path}/{result_path}"):
35+
os.makedirs(f"{root_path}/{result_path}")
36+
37+
# %%
38+
stations = pd.read_csv(f"{root_path}/{data_path}/cctorch_stations.csv")
39+
40+
station_lines = {}
41+
for i, row in stations.iterrows():
42+
station_id = row["station_id"]
43+
network_code, station_code, comp_code, channel_code = station_id.split(".")
44+
# tmp_code = f"{station_code}{channel_code}"
45+
tmp_code = f"{station_code}"
46+
station_lines[tmp_code] = f"{tmp_code:<8s} {row['latitude']:.3f} {row['longitude']:.3f}\n"
47+
48+
49+
with open(f"{root_path}/{result_path}/stations.dat", "w") as f:
50+
for line in sorted(station_lines.values()):
51+
f.write(line)
52+
53+
# %%
54+
events = pd.read_csv(f"{root_path}/{data_path}/cctorch_events.csv")
55+
events["time"] = pd.to_datetime(events["event_time"], format="mixed")
56+
57+
event_lines = []
58+
59+
for i, row in events.iterrows():
60+
event_index = row["event_index"]
61+
origin = row["time"]
62+
magnitude = row["magnitude"]
63+
x_err = 0.0
64+
z_err = 0.0
65+
time_err = 0.0
66+
dx, dy, dz = 0.0, 0.0, 0.0
67+
# dx = np.random.uniform(-0.01, 0.01)
68+
# dy = np.random.uniform(-0.01, 0.01)
69+
# dz = np.random.uniform(0, 10)
70+
# dz = 0
71+
event_lines.append(
72+
f"{origin.year:4d}{origin.month:02d}{origin.day:02d} "
73+
f"{origin.hour:2d}{origin.minute:02d}{origin.second:02d}{round(origin.microsecond / 1e4):02d} "
74+
# f"{row['latitude']:8.4f} {row['longitude']:9.4f} {row['depth_km']:8.4f} "
75+
f"{row['latitude'] + dy:8.4f} {row['longitude']+ dx:9.4f} {row['depth_km']+dz:8.4f} "
76+
f"{magnitude:5.2f} {x_err:5.2f} {z_err:5.2f} {time_err:5.2f} {event_index:9d}\n"
77+
)
78+
79+
with open(f"{root_path}/{result_path}/events.dat", "w") as f:
80+
f.writelines(event_lines)
81+
82+
# %%
83+
os.system(f"bash run_hypodd_cc.sh {root_path} {region}")

‎examples/california/run_hypodd_cc.sh‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
#!/bin/bash
2+
set -x
3+
WORKING_DIR=$PWD
4+
if [ $# -eq 2 ]; then
5+
root_path=$1
6+
region=$2
7+
else
8+
root_path="local"
9+
region="demo"
10+
fi
11+
12+
if [ ! -d "$root_path/$region/hypodd" ]; then
13+
mkdir -p $root_path/$region/hypodd
14+
fi
15+
16+
cp $root_path/$region/cctorch/dt.cc $root_path/$region/hypodd/dt.cc
17+
cd $root_path/$region/hypodd
18+
19+
if [ ! -d "HypoDD" ]; then
20+
git clone [email protected]:zhuwq0/HypoDD.git
21+
export PATH=$PATH:$PWD/HypoDD
22+
make -C HypoDD/src/
23+
fi
24+
25+
cat <<EOF > cc.inp
26+
* RELOC.INP:
27+
*--- input file selection
28+
* cross correlation diff times:
29+
dt.cc
30+
*
31+
*catalog P diff times:
32+
33+
*
34+
* event file:
35+
events.dat
36+
*
37+
* station file:
38+
stations.dat
39+
*
40+
*--- output file selection
41+
* original locations:
42+
hypodd_cc.loc
43+
* relocations:
44+
hypodd_cc.reloc
45+
* station information:
46+
hypodd.sta
47+
* residual information:
48+
hypodd.res
49+
* source paramater information:
50+
hypodd.src
51+
*
52+
*--- data type selection:
53+
* IDAT: 0 = synthetics; 1= cross corr; 2= catalog; 3= cross & cat
54+
* IPHA: 1= P; 2= S; 3= P&S
55+
* DIST:max dist [km] between cluster centroid and station
56+
* IDAT IPHA DIST
57+
1 3 120
58+
*
59+
*--- event clustering:
60+
* OBSCC: min # of obs/pair for crosstime data (0= no clustering)
61+
* OBSCT: min # of obs/pair for network data (0= no clustering)
62+
* OBSCC OBSCT
63+
0 0
64+
*
65+
*--- solution control:
66+
* ISTART: 1 = from single source; 2 = from network sources
67+
* ISOLV: 1 = SVD, 2=lsqr
68+
* NSET: number of sets of iteration with specifications following
69+
* ISTART ISOLV NSET
70+
2 2 4
71+
*
72+
*--- data weighting and re-weighting:
73+
* NITER: last iteration to used the following weights
74+
* WTCCP, WTCCS: weight cross P, S
75+
* WTCTP, WTCTS: weight catalog P, S
76+
* WRCC, WRCT: residual threshold in sec for cross, catalog data
77+
* WDCC, WDCT: max dist [km] between cross, catalog linked pairs
78+
* DAMP: damping (for lsqr only)
79+
* --- CROSS DATA ----- ----CATALOG DATA ----
80+
* NITER WTCCP WTCCS WRCC WDCC WTCTP WTCTS WRCT WDCT DAMP
81+
4 1 1 -9 -9 -9 -9 -9 -9 70
82+
4 1 1 6 -9 -9 -9 -9 -9 70
83+
4 1 0.8 3 4 -9 -9 -9 -9 70
84+
4 1 0.8 2 2 -9 -9 -9 -9 70
85+
*
86+
*--- 1D model:
87+
* NLAY: number of model layers
88+
* RATIO: vp/vs ratio
89+
* TOP: depths of top of layer (km)
90+
* VEL: layer velocities (km/s)
91+
* NLAY RATIO
92+
12 1.73
93+
* TOP
94+
0.0 1.0 3.0 5.0 7.0 9.0 11.0 13.0 17.0 21.0 31.00 31.10
95+
* VEL
96+
5.30 5.65 5.93 6.20 6.20 6.20 6.20 6.20 6.20 6.20 7.50 8.11
97+
*
98+
*--- event selection:
99+
* CID: cluster to be relocated (0 = all)
100+
* ID: cuspids of event to be relocated (8 per line)
101+
* CID
102+
0
103+
* ID
104+
EOF
105+
106+
./HypoDD/src/hypoDD/hypoDD cc.inp
107+
cd $WORKING_DIR

0 commit comments

Comments
 (0)