From b900cd1e1dfa86fefb729d678d02656c4a224c4c Mon Sep 17 00:00:00 2001 From: aditigopalan <63365451+aditigopalan@users.noreply.github.com> Date: Tue, 28 Jan 2025 16:37:17 -0500 Subject: [PATCH 1/7] Adding git action for rfc sync --- .github/workflows/google-sheet-sync.yml | 47 +++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .github/workflows/google-sheet-sync.yml diff --git a/.github/workflows/google-sheet-sync.yml b/.github/workflows/google-sheet-sync.yml new file mode 100644 index 00000000..62d389e0 --- /dev/null +++ b/.github/workflows/google-sheet-sync.yml @@ -0,0 +1,47 @@ +name: "Google Sheet Sync" + +on: + schedule: + - cron: "0 0 * * *" # Run once a day at midnight UTC + workflow_dispatch: + inputs: + spreadsheet_id: + description: "The ID of the Google Sheet to monitor." + required: true + stop_time: + description: "The UTC time (HH:MM) at which the action should stop." + required: true + stop_day: + description: "The day (YYYY-MM-DD) on which the action should stop." + required: true + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - name: Check out the repository + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install google-api-python-client PyGithub google-auth + + - name: Create Google Credentials File + env: + GOOGLE_CREDENTIALS: ${{ secrets.GOOGLE_CREDENTIALS }} + run: | + echo "${GOOGLE_CREDENTIALS}" > google-credentials.json + + - name: Run Google Sheet Sync + env: + INPUT_SPREADSHEET_ID: ${{ github.event.inputs.spreadsheet_id }} + INPUT_STOP_TIME: ${{ github.event.inputs.stop_time }} + INPUT_STOP_DAY: ${{ github.event.inputs.stop_day }} + GITHUB_TOKEN: ${{ secrets.RFC_TOKEN }} + run: python google-sheet-sync/sync.py \ No newline at end of file From 7af70c86986b6df5686185660449462256956cae Mon Sep 17 00:00:00 2001 From: aditigopalan <63365451+aditigopalan@users.noreply.github.com> Date: Tue, 28 Jan 2025 16:37:33 -0500 Subject: [PATCH 2/7] Adding input specifications for rfc --- google-sheet-sync/action.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 google-sheet-sync/action.yml diff --git a/google-sheet-sync/action.yml b/google-sheet-sync/action.yml new file mode 100644 index 00000000..b04cbc06 --- /dev/null +++ b/google-sheet-sync/action.yml @@ -0,0 +1,15 @@ +name: "Google Sheet Sync for RFC Integration" +description: "Syncs a Google Sheet with a GitHub repository as a CSV for the purpose of RFC Integration." +inputs: + spreadsheet_id: + description: "The ID of the Google Sheet to monitor." + required: true + stop_time: + description: "The UTC time (HH:MM) at which the action should stop." + required: true + stop_day: + description: "The day (YYYY-MM-DD) on which the action should stop." + required: true +runs: + using: "python" + main: "sync.py" \ No newline at end of file From e2dfdaca431754e23f3f6c1b57a33c9b69bf683e Mon Sep 17 00:00:00 2001 From: aditigopalan <63365451+aditigopalan@users.noreply.github.com> Date: Tue, 28 Jan 2025 16:37:41 -0500 Subject: [PATCH 3/7] Adding sync script --- google-sheet-sync/sync.py | 97 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 google-sheet-sync/sync.py diff --git a/google-sheet-sync/sync.py b/google-sheet-sync/sync.py new file mode 100644 index 00000000..a96e7c86 --- /dev/null +++ b/google-sheet-sync/sync.py @@ -0,0 +1,97 @@ +import os +import csv +from io import StringIO +from datetime import datetime +from google.oauth2.service_account import Credentials +from googleapiclient.discovery import build +from github import Github + +def authenticate_google(credentials_file): + creds = Credentials.from_service_account_file(credentials_file, scopes=["https://www.googleapis.com/auth/spreadsheets.readonly"]) + return build('sheets', 'v4', credentials=creds) + +def get_sheet_name(service, spreadsheet_id): + sheet_metadata = service.spreadsheets().get(spreadsheetId=spreadsheet_id).execute() + return sheet_metadata.get("properties", {}).get("title", "UntitledSheet").replace(" ", "-") + +def get_sheet_data_as_csv(service, spreadsheet_id, range="A1:Z1000"): + sheet = service.spreadsheets() + result = sheet.values().get(spreadsheetId=spreadsheet_id, range=range).execute() + data = result.get('values', []) + if not data: + return None + output = StringIO() + writer = csv.writer(output) + writer.writerows(data) + output.seek(0) + return output.read() + +def main(): + + spreadsheet_id = os.getenv("INPUT_SPREADSHEET_ID") + stop_time = os.getenv("INPUT_STOP_TIME") + stop_day = os.getenv("INPUT_STOP_DAY") + github_token = os.getenv("GITHUB_TOKEN") + repo_name = os.getenv("GITHUB_REPOSITORY") + credentials_file = "./google-credentials.json" # Path to dynamically created credentials file + + + service = authenticate_google(credentials_file) + + sheet_name = get_sheet_name(service, spreadsheet_id) + branch_name = f"RFC-{sheet_name}" + file_path = f"RFC_DATA/{sheet_name}.csv" + + github = Github(github_token) + repo = github.get_repo(repo_name) + + current_day = datetime.utcnow().strftime("%Y-%m-%d") + current_time = datetime.utcnow().strftime("%H:%M") + + + if current_day > stop_day or (current_day == stop_day and current_time > stop_time): + print(f"Current day/time ({current_day} {current_time}) has exceeded the stop conditions ({stop_day} {stop_time}). Exiting.") + return + + + csv_content = get_sheet_data_as_csv(service, spreadsheet_id) + if not csv_content: + print("No data fetched from Google Sheet. Exiting.") + return + + # Ensure branch exists + try: + repo.get_branch(branch_name) + except Exception: + main_branch = repo.get_branch("main") + repo.create_git_ref(ref=f"refs/heads/{branch_name}", sha=main_branch.commit.sha) + + # Commit changes to the branch + try: + contents = None + try: + contents = repo.get_contents(file_path, ref=branch_name) + except Exception: + pass + if contents: + repo.update_file(file_path, "Update CSV file from Google Sheet", csv_content, contents.sha, branch=branch_name) + else: + repo.create_file(file_path, "Create CSV file from Google Sheet", csv_content, branch=branch_name) + except Exception as e: + print(f"Error committing changes: {e}") + + # Create or reuse PR + try: + open_prs = repo.get_pulls(state="open", head=f"{repo.owner.login}:{branch_name}") + if open_prs.totalCount == 0: + repo.create_pull( + title=f"{branch_name} Updates", + body="This PR contains updates from the linked Google Sheet.", + head=branch_name, + base="main", + ) + except Exception as e: + print(f"Error creating PR: {e}") + +if __name__ == "__main__": + main() \ No newline at end of file From 009f698778ad861cadbc2ac4b67b0d2fdde957e2 Mon Sep 17 00:00:00 2001 From: Aditi <63365451+aditigopalan@users.noreply.github.com> Date: Tue, 4 Feb 2025 12:58:55 -0500 Subject: [PATCH 4/7] Update sync.py --- google-sheet-sync/sync.py | 63 +++++++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 15 deletions(-) diff --git a/google-sheet-sync/sync.py b/google-sheet-sync/sync.py index a96e7c86..f4cf1255 100644 --- a/google-sheet-sync/sync.py +++ b/google-sheet-sync/sync.py @@ -4,9 +4,25 @@ from datetime import datetime from google.oauth2.service_account import Credentials from googleapiclient.discovery import build -from github import Github +from github import Github, GithubException def authenticate_google(credentials_file): + """ + Authenticate and create a Google Sheets API client. + + This function reads a Google service account credentials file to authenticate + and build a client for accessing the Google Sheets API. + Args: + credentials_file (str): The path to the Google service account JSON credentials file. + GOOGLE_CREDENTIALS stored securely as GitHub Secret on the mc2-centerbot account + + Returns: + googleapiclient.discovery.Resource: An authenticated client object for interacting + with the Google Sheets API. + + Raises: + google.auth.exceptions.GoogleAuthError: If authentication fails due to invalid or missing credentials. + """ creds = Credentials.from_service_account_file(credentials_file, scopes=["https://www.googleapis.com/auth/spreadsheets.readonly"]) return build('sheets', 'v4', credentials=creds) @@ -62,24 +78,41 @@ def main(): # Ensure branch exists try: repo.get_branch(branch_name) - except Exception: + except GithubException as e: + # Check if the error is due to a non-existent branch + if e.status == 404: main_branch = repo.get_branch("main") repo.create_git_ref(ref=f"refs/heads/{branch_name}", sha=main_branch.commit.sha) + else: + raise - # Commit changes to the branch +# Commit changes to the branch +try: try: - contents = None - try: - contents = repo.get_contents(file_path, ref=branch_name) - except Exception: - pass - if contents: - repo.update_file(file_path, "Update CSV file from Google Sheet", csv_content, contents.sha, branch=branch_name) + # Attempt to get the file contents + contents = repo.get_contents(file_path, ref=branch_name) + # Update the file if it exists + repo.update_file( + file_path, + "Update CSV file from Google Sheet", + csv_content, + contents.sha, + branch=branch_name + ) + except GithubException as e: + # If the file doesn't exist (404), create it + if e.status == 404: + repo.create_file( + file_path, + "Create CSV file from Google Sheet", + csv_content, + branch=branch_name + ) else: - repo.create_file(file_path, "Create CSV file from Google Sheet", csv_content, branch=branch_name) - except Exception as e: - print(f"Error committing changes: {e}") - + raise # Re-raise other exceptions +except Exception as e: + print(f"Error committing changes: {e}") + # Create or reuse PR try: open_prs = repo.get_pulls(state="open", head=f"{repo.owner.login}:{branch_name}") @@ -94,4 +127,4 @@ def main(): print(f"Error creating PR: {e}") if __name__ == "__main__": - main() \ No newline at end of file + main() From 7c0b9bb1e5437276220d0c9d0933a25d3b1f7b82 Mon Sep 17 00:00:00 2001 From: Aditi <63365451+aditigopalan@users.noreply.github.com> Date: Thu, 13 Feb 2025 13:35:53 -0500 Subject: [PATCH 5/7] Update google-sheet-sync/sync.py Co-authored-by: Verena Chung <9377970+vpchung@users.noreply.github.com> --- google-sheet-sync/sync.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/google-sheet-sync/sync.py b/google-sheet-sync/sync.py index f4cf1255..854222a6 100644 --- a/google-sheet-sync/sync.py +++ b/google-sheet-sync/sync.py @@ -20,8 +20,6 @@ def authenticate_google(credentials_file): googleapiclient.discovery.Resource: An authenticated client object for interacting with the Google Sheets API. - Raises: - google.auth.exceptions.GoogleAuthError: If authentication fails due to invalid or missing credentials. """ creds = Credentials.from_service_account_file(credentials_file, scopes=["https://www.googleapis.com/auth/spreadsheets.readonly"]) return build('sheets', 'v4', credentials=creds) From 64bb384869b5eef93489e3a3256bccec5eb2640d Mon Sep 17 00:00:00 2001 From: Aditi <63365451+aditigopalan@users.noreply.github.com> Date: Thu, 13 Feb 2025 13:39:31 -0500 Subject: [PATCH 6/7] Update google-sheet-sync/sync.py Co-authored-by: Verena Chung <9377970+vpchung@users.noreply.github.com> --- google-sheet-sync/sync.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/google-sheet-sync/sync.py b/google-sheet-sync/sync.py index 854222a6..1d84f4ee 100644 --- a/google-sheet-sync/sync.py +++ b/google-sheet-sync/sync.py @@ -6,19 +6,17 @@ from googleapiclient.discovery import build from github import Github, GithubException -def authenticate_google(credentials_file): - """ - Authenticate and create a Google Sheets API client. +def authenticate_google(credentials_file: str) -> googleapiclient.discovery.Resource: + """Authenticate and create a Google Sheets API client. This function reads a Google service account credentials file to authenticate and build a client for accessing the Google Sheets API. + Args: - credentials_file (str): The path to the Google service account JSON credentials file. - GOOGLE_CREDENTIALS stored securely as GitHub Secret on the mc2-centerbot account + credentials_file: The path to the Google service account JSON credentials file. Returns: - googleapiclient.discovery.Resource: An authenticated client object for interacting - with the Google Sheets API. + An authenticated client object for interacting with the Google Sheets API. """ creds = Credentials.from_service_account_file(credentials_file, scopes=["https://www.googleapis.com/auth/spreadsheets.readonly"]) From 83242a8c9575322638b3557bf4ef383bf9f34e92 Mon Sep 17 00:00:00 2001 From: Aditi <63365451+aditigopalan@users.noreply.github.com> Date: Thu, 13 Feb 2025 13:49:07 -0500 Subject: [PATCH 7/7] Update sync.py --- google-sheet-sync/sync.py | 98 ++++++++++++++++++++++----------------- 1 file changed, 55 insertions(+), 43 deletions(-) diff --git a/google-sheet-sync/sync.py b/google-sheet-sync/sync.py index 1d84f4ee..06156274 100644 --- a/google-sheet-sync/sync.py +++ b/google-sheet-sync/sync.py @@ -6,27 +6,45 @@ from googleapiclient.discovery import build from github import Github, GithubException -def authenticate_google(credentials_file: str) -> googleapiclient.discovery.Resource: - """Authenticate and create a Google Sheets API client. +def authenticate_google(credentials_file: str) -> build: + """ + Authenticate and create a Google Sheets API client. - This function reads a Google service account credentials file to authenticate - and build a client for accessing the Google Sheets API. - Args: - credentials_file: The path to the Google service account JSON credentials file. + credentials_file (str): Path to the Google service account JSON credentials file. Returns: - An authenticated client object for interacting with the Google Sheets API. - + googleapiclient.discovery.Resource: Authenticated client for interacting with Google Sheets API. """ creds = Credentials.from_service_account_file(credentials_file, scopes=["https://www.googleapis.com/auth/spreadsheets.readonly"]) return build('sheets', 'v4', credentials=creds) -def get_sheet_name(service, spreadsheet_id): +def get_sheet_name(service, spreadsheet_id: str) -> str: + """ + Retrieve the title of the Google Sheet and format it for use as a branch name. + + Args: + service (googleapiclient.discovery.Resource): Authenticated Google Sheets API client. + spreadsheet_id (str): ID of the Google Sheet. + + Returns: + str: Formatted sheet title with spaces replaced by dashes. + """ sheet_metadata = service.spreadsheets().get(spreadsheetId=spreadsheet_id).execute() return sheet_metadata.get("properties", {}).get("title", "UntitledSheet").replace(" ", "-") -def get_sheet_data_as_csv(service, spreadsheet_id, range="A1:Z1000"): +def get_sheet_data_as_csv(service, spreadsheet_id: str, range: str = "A1:Z1000") -> str: + """ + Retrieve data from the specified Google Sheet range and return it as a CSV string. + + Args: + service (googleapiclient.discovery.Resource): Authenticated Google Sheets API client. + spreadsheet_id (str): ID of the Google Sheet. + range (str, optional): Cell range to fetch data from. Defaults to "A1:Z1000". + + Returns: + str: CSV formatted data from the sheet, or None if the sheet is empty. + """ sheet = service.spreadsheets() result = sheet.values().get(spreadsheetId=spreadsheet_id, range=range).execute() data = result.get('values', []) @@ -39,7 +57,6 @@ def get_sheet_data_as_csv(service, spreadsheet_id, range="A1:Z1000"): return output.read() def main(): - spreadsheet_id = os.getenv("INPUT_SPREADSHEET_ID") stop_time = os.getenv("INPUT_STOP_TIME") stop_day = os.getenv("INPUT_STOP_DAY") @@ -47,9 +64,7 @@ def main(): repo_name = os.getenv("GITHUB_REPOSITORY") credentials_file = "./google-credentials.json" # Path to dynamically created credentials file - service = authenticate_google(credentials_file) - sheet_name = get_sheet_name(service, spreadsheet_id) branch_name = f"RFC-{sheet_name}" file_path = f"RFC_DATA/{sheet_name}.csv" @@ -60,12 +75,10 @@ def main(): current_day = datetime.utcnow().strftime("%Y-%m-%d") current_time = datetime.utcnow().strftime("%H:%M") - if current_day > stop_day or (current_day == stop_day and current_time > stop_time): print(f"Current day/time ({current_day} {current_time}) has exceeded the stop conditions ({stop_day} {stop_time}). Exiting.") return - csv_content = get_sheet_data_as_csv(service, spreadsheet_id) if not csv_content: print("No data fetched from Google Sheet. Exiting.") @@ -75,39 +88,38 @@ def main(): try: repo.get_branch(branch_name) except GithubException as e: - # Check if the error is due to a non-existent branch - if e.status == 404: - main_branch = repo.get_branch("main") - repo.create_git_ref(ref=f"refs/heads/{branch_name}", sha=main_branch.commit.sha) - else: - raise - -# Commit changes to the branch -try: - try: - # Attempt to get the file contents - contents = repo.get_contents(file_path, ref=branch_name) - # Update the file if it exists - repo.update_file( - file_path, - "Update CSV file from Google Sheet", - csv_content, - contents.sha, - branch=branch_name - ) - except GithubException as e: - # If the file doesn't exist (404), create it if e.status == 404: - repo.create_file( + main_branch = repo.get_branch("main") + repo.create_git_ref(ref=f"refs/heads/{branch_name}", sha=main_branch.commit.sha) + else: + raise + + # Commit changes to the branch + try: + try: + # Attempt to get the file contents + contents = repo.get_contents(file_path, ref=branch_name) + # Update the file if it exists + repo.update_file( file_path, - "Create CSV file from Google Sheet", + "Update CSV file from Google Sheet", csv_content, + contents.sha, branch=branch_name ) - else: - raise # Re-raise other exceptions -except Exception as e: - print(f"Error committing changes: {e}") + except GithubException as e: + # If the file doesn't exist (404), create it + if e.status == 404: + repo.create_file( + file_path, + "Create CSV file from Google Sheet", + csv_content, + branch=branch_name + ) + else: + raise # Re-raise other exceptions + except Exception as e: + print(f"Error committing changes: {e}") # Create or reuse PR try: