v0.6.0

docs/reference/changelog.md

@@ -1,5 +1,10 @@
 # Changelog
 
+0.6.0
+
+* All currency values use decimal.Decimal.
+* Handle Rakuten ETC card charges in Rakuten Card.
+
 0.5.0
 
 * Add support for SBI Sumishin Net Bank.

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "beancount-multitool"
-version = "0.5.0"
+version = "0.6.0"
 description = "A CLI tool that converts financial data to Beancount files"
 authors = ["Rick Lan <[email protected]>"]
 license = "MIT"

src/beancount_multitool/JABank.py

@@ -1,8 +1,8 @@
 from datetime import datetime
-import pandas as pd
+from decimal import Decimal
 from pathlib import Path
-import uuid
-import sys
+
+import pandas as pd
 
 from .Institution import Institution
 from .MappingDatabase import MappingDatabase
@@ -43,7 +43,17 @@ def read_transaction(self, file_name: str, year: int) -> pd.DataFrame:
         pd.DataFrame
             A dataframe after pre-processing.
         """
-        df = pd.read_csv(file_name, encoding="shift_jis_2004")
+        converters = {
+            "明細区分": str,
+            "取扱日付": str,
+            "起算日": str,
+            "お支払金額": str,
+            "お預り金額": str,
+            "取引区分": str,
+            "残高": str,
+            "摘要": str,
+        }
+        df = pd.read_csv(file_name, encoding="shift_jis_2004", converters=converters)
         print(f"Found {len(df.index)} transactions in {file_name}")
 
         # Rename column names to English
@@ -66,13 +76,21 @@ def read_transaction(self, file_name: str, year: int) -> pd.DataFrame:
             str(year) + "." + df["Handling Date"], format="%Y.%m月%d日"
         )
 
-        cols = ["Debit", "Credit"]
+        cols = ["Debit", "Credit", "Balance"]
         df[cols] = df[cols].replace({"\¥": "", ",": ""}, regex=True)
-        df.fillna({"Debit": 0, "Credit": 0}, inplace=True)
-        # Convert float to int
-        df[cols] = df[cols].astype(int)
+        df.replace(
+            to_replace="",
+            value={"Debit": "0", "Credit": "0", "Balance": "0"},
+            inplace=True,
+        )
+
+        df["Debit"] = df["Debit"].apply(Decimal)
+        df["Credit"] = df["Credit"].apply(Decimal)
+        df["Balance"] = df["Balance"].apply(Decimal)
+
         df["amount"] = df["Credit"] - df["Debit"]
         df["memo"] = df["Transaction Classification"] + df["Description"]
+
         # print(df.dtypes) # debug
         # print(df) # debug
         return df

src/beancount_multitool/RakutenBank.py

@@ -1,7 +1,7 @@
-import pandas as pd
+from decimal import Decimal
 from pathlib import Path
-import uuid
-import sys
+
+import pandas as pd
 
 from .Institution import Institution
 from .MappingDatabase import MappingDatabase
@@ -42,7 +42,12 @@ def read_transaction(self, file_name: str) -> pd.DataFrame:
         pd.DataFrame
             A dataframe after pre-processing.
         """
-        df = pd.read_csv(file_name, encoding="shiftjis")
+        converters = {
+            "取引日": pd.to_datetime,
+            "入出金(円)": str,
+            "取引後残高(円)": str,
+        }
+        df = pd.read_csv(file_name, encoding="shift_jis", converters=converters)
         print(f"Found {len(df.index)} transactions in {file_name}")
 
         # Rename column names to English.
@@ -56,8 +61,9 @@ def read_transaction(self, file_name: str) -> pd.DataFrame:
         }
         df.rename(columns=column_names, inplace=True)
 
-        # Convert date column to a datetime object
-        df["date"] = pd.to_datetime(df["date"], format="%Y%m%d")
+        df["amount"] = df["amount"].apply(Decimal)
+        df["Balance"] = df["Balance"].apply(Decimal)
+
         # print(df.dtypes) # debug
         # print(df) # debug
         return df

src/beancount_multitool/RakutenCard.py

@@ -1,6 +1,8 @@
-import pandas as pd
+from decimal import Decimal
 from pathlib import Path
 
+import pandas as pd
+
 from .Institution import Institution
 from .MappingDatabase import MappingDatabase
 from .read_config import read_config
@@ -37,7 +39,16 @@ def read_transaction(self, file_name: str) -> pd.DataFrame:
         pd.DataFrame
             A dataframe after pre-processing.
         """
-        df = pd.read_csv(file_name)
+        converters = {
+            "利用日": pd.to_datetime,
+            "利用店名・商品名": str,
+            "利用者": str,
+            # "支払方法": "Payment method",
+            # "利用金額": "Amount",
+            # "支払手数料": "Commission paid",
+            "支払総額": str,
+        }
+        df = pd.read_csv(file_name, converters=converters)
         print(f"Found {len(df.index)} transactions in {file_name}")
 
         # Rename column names to English.
@@ -55,20 +66,25 @@ def read_transaction(self, file_name: str) -> pd.DataFrame:
         }
         df.rename(columns=column_names, inplace=True)
 
-        df["date"] = pd.to_datetime(df["date"], format="%Y/%m/%d")
-
-        # Remove rows with empty 支払総額 cell.
-        # These are extra info such as name of ETC gate or currency exchange rate.
-        # TODO record as metadata
-        extra = df.loc[pd.isnull(df["amount"])]
-        df.drop(extra.index, inplace=True)
-        # Convert to int type because currency is JPY
-        df = df.astype({"amount": int})
+        # ETC transaction has a second row.
+        # Update firsts to be concatenation of the two memos.
+        # Then delete the seconds.
+        etc_index = df.loc[df["user"] == "ETC"].index
+        df.loc[etc_index, "memo"] = (
+            df.loc[etc_index, "memo"].values
+            + " "
+            + df.loc[etc_index + 1, "memo"].values
+        )
+        df.drop(df.loc[df["user"] == ""].index, inplace=True)
 
         # Remove rows with zero 支払総額. These are refunds.
+        # Also currency exchange rate(?)
         # TODO record as metadata.
-        refund = df.loc[df["amount"] == 0]
-        df.drop(refund.index, inplace=True)
+        # refund = df.loc[df["amount"] == 0]
+        # df.drop(refund.index, inplace=True)
+
+        # TODO this will fail if the refund rows are not removed
+        df["amount"] = df["amount"].apply(Decimal)
 
         # Reverse row order because the oldest transaction is on the bottom
         # Note: the index column is also reversed

src/beancount_multitool/ShinseiBank.py

@@ -1,7 +1,7 @@
-import pandas as pd
+from decimal import Decimal
 from pathlib import Path
-import uuid
-import sys
+
+import pandas as pd
 
 from .Institution import Institution
 from .MappingDatabase import MappingDatabase
@@ -42,7 +42,17 @@ def read_transaction(self, file_name: str) -> pd.DataFrame:
         pd.DataFrame
             A dataframe after pre-processing.
         """
-        df = pd.read_csv(file_name)
+        converters = {
+            "取引日": pd.to_datetime,
+            "出金金額": str,
+            "入金金額": str,
+            "残高": str,
+            "Value Date": pd.to_datetime,
+            "Debit": str,
+            "Credit": str,
+            "Balance": str,
+        }
+        df = pd.read_csv(file_name, converters=converters)
         print(f"Found {len(df.index)} transactions in {file_name}")
 
         # Rename column names to English.
@@ -64,13 +74,24 @@ def read_transaction(self, file_name: str) -> pd.DataFrame:
         }
         df.rename(columns=column_names, inplace=True)
 
-        # Convert date column to a datetime object
-        df["date"] = pd.to_datetime(df["date"], format="%Y/%m/%d")
         # Fill empty cells with zeros
-        df.fillna(0, inplace=True)
-        # Convert float to int
-        df["Debit"] = df["Debit"].astype(int)
-        df["Credit"] = df["Credit"].astype(int)
+        df.replace(
+            to_replace="",
+            value={"Debit": "0", "Credit": "0", "Balance": "0"},
+            inplace=True,
+        )
+        # Remove thousands marker
+        df.replace(
+            to_replace=",",
+            value={"Debit": "", "Credit": "", "Balance": ""},
+            inplace=True,
+            regex=True,
+        )
+
+        df["Debit"] = df["Debit"].apply(Decimal)
+        df["Credit"] = df["Credit"].apply(Decimal)
+        df["Balance"] = df["Balance"].apply(Decimal)
+
         df["amount"] = df["Credit"] - df["Debit"]
 
         # Reverse row order because the oldest transaction is on the bottom

src/beancount_multitool/SumishinNetBank.py

@@ -1,7 +1,5 @@
 from decimal import Decimal
 from pathlib import Path
-import uuid
-import sys
 
 import pandas as pd
 
@@ -44,14 +42,13 @@ def read_transaction(self, file_name: str) -> pd.DataFrame:
         pd.DataFrame
             A dataframe after pre-processing.
         """
-        df = pd.read_csv(file_name, 
-                        encoding="shift_jis",
-                        converters={
-                            "日付": pd.to_datetime,
-                            "出金金額(円)": str,
-                            "入金金額(円)": str,
-                            "残高(円)": str,
-        })
+        converters = {
+            "日付": pd.to_datetime,
+            "出金金額(円)": str,
+            "入金金額(円)": str,
+            "残高(円)": str,
+        }
+        df = pd.read_csv(file_name, encoding="shift_jis", converters=converters)
         print(f"Found {len(df.index)} data transactions")
         # Rename column names to English.
         # "日付","内容","出金金額(円)","入金金額(円)","残高(円)","メモ"
@@ -67,8 +64,17 @@ def read_transaction(self, file_name: str) -> pd.DataFrame:
         }
         df.rename(columns=columns, inplace=True)
 
-        df.replace(to_replace="", value={"Debit": "0", "Credit": "0", "Balance": "0"}, inplace=True)
-        df.replace(to_replace=",", value={"Debit": "", "Credit": "", "Balance": ""}, inplace=True, regex=True)
+        df.replace(
+            to_replace="",
+            value={"Debit": "0", "Credit": "0", "Balance": "0"},
+            inplace=True,
+        )
+        df.replace(
+            to_replace=",",
+            value={"Debit": "", "Credit": "", "Balance": ""},
+            inplace=True,
+            regex=True,
+        )
         df["Debit"] = df["Debit"].apply(Decimal)
         df["Credit"] = df["Credit"].apply(Decimal)
         df["Balance"] = df["Balance"].apply(Decimal)

src/beancount_multitool/__version__.py

@@ -1 +1 @@
-__version__ = "0.5.0"
+__version__ = "0.6.0"

src/beancount_multitool/as_transaction.py

@@ -5,10 +5,9 @@
 
 
 def make_hashtags(func):
-    """Add # prefix if missing from any tags.
-    """
-    def add_hash(*args, **kwargs):
+    """Add # prefix if missing from any tags."""
 
+    def add_hash(*args, **kwargs):
         hashtags = []
         for x in kwargs["tags"]:
             if len(x):
@@ -20,14 +19,14 @@ def add_hash(*args, **kwargs):
 
         result = func(*args, **kwargs)
         return result
+
     return add_hash
 
 
 def reconcile(func):
-    """Add an UUID field if #reconcile tag exists
-    """
-    def add_uuid(*args, **kwargs):
+    """Add an UUID field if #reconcile tag exists"""
 
+    def add_uuid(*args, **kwargs):
         # Add UUID for manual transactions reconcilation between accounts
         if "#reconcile" in kwargs["tags"]:
             if kwargs["amount"] < 0:  # a credit
@@ -41,6 +40,7 @@ def add_uuid(*args, **kwargs):
 
         result = func(*args, **kwargs)
         return result
+
     return add_uuid
 
 

src/beancount_multitool/cli.py

@@ -19,9 +19,7 @@ def validate_name(ctx, param, value):
         raise click.BadParameter(f"Name must be one of: {__INSTITUTIONS__}")
 
 
-@click.command(
-    epilog=f"Note: supported financial institutions are {__INSTITUTIONS__}"
-)
+@click.command(epilog=f"Note: supported financial institutions are {__INSTITUTIONS__}")
 @click.argument("name", type=str, callback=validate_name)
 @click.argument("config", type=click.Path(exists=True))
 @click.argument("data", type=click.Path(exists=True))

src/beancount_multitool/tests/test_read_config.py

@@ -1,4 +1,3 @@
-#import pytest
 from beancount_multitool.read_config import read_config
 
 

tests/conftest.py

@@ -3,9 +3,12 @@
 # Ref: https://stackoverflow.com/a/25188424
 def pytest_configure(config):
     import sys
+
     sys._called_from_pytest = True
 
+
 def pytest_unconfigure(config):
     import sys
+
     if hasattr(sys, "_called_from_pytest"):
         del sys._called_from_pytest