File tree Expand file tree Collapse file tree
Expand file tree Collapse file tree Original file line number Diff line number Diff line change @@ -194,8 +194,12 @@ def remove_timeseries_with_duplicated(df: pd.DataFrame) -> pd.DataFrame:
194194 # The pc_missing the less the better, so we only keep the first one
195195 df = df .drop_duplicates ("DAY" , keep = "first" )
196196
197- # Remove entries within 60s, this takes care of the edge case where duplicates wrap across midnight UTC
198- df ['TIMEDIFF' ] = pd .to_datetime (df ['date' ].shift (- 1 )) - pd .to_datetime (df ['date' ])
197+ # Remove entries within 60s, deals with edge cases where duplicates wrap across midnight UTC
198+
199+ if "date" not in df .columns :
200+ df ['TIMEDIFF' ] = pd .to_datetime (df .index .shift (- 1 )) - pd .to_datetime (df .index )
201+ else :
202+ df ['TIMEDIFF' ] = pd .to_datetime (df ['date' ].shift (- 1 )) - pd .to_datetime (df ['date' ])
199203 df = df [~ (df ['TIMEDIFF' ] < timedelta (seconds = 60 ))]
200204 df = df .drop (columns = ["TIMEDIFF" ])
201205
You can’t perform that action at this time.
0 commit comments