diff --git a/.gitignore b/.gitignore index f5f4011..e7ace45 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .env +.env.local .playwright-mcp/ CLAUDE.md SECURITYPLAN.md diff --git a/README.md b/README.md index 9bba0c0..c9d0cf3 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ All runtime configuration lives in `.env`. Edit it before building: | `NODE_VERSION_YCMD` | `16.20.2` | Node.js version in ycmd image | | `NGINX_PORT` | `8080` | Host port nginx binds to | | `PUBLIC_URL` | `http://localhost:8080` | Public-facing URL (see note below) | +| `GITHUB_HOOK_URL` | *(uses PUBLIC_URL)* | Base URL for GitHub webhook callbacks (see ngrok note below) | | `EXPECT_SSL` | `no` | Set to `yes` for HTTPS deployments | | `EMULATOR_FIXED_LIMIT` | `90` | Max concurrent emulators | | `LIBPEBBLE_PROXY` | `wss://cloudpebble-proxy.repebble.com/tool` | libpebble proxy WebSocket URL | @@ -37,6 +38,29 @@ All runtime configuration lives in `.env`. Edit it before building: > **Note:** `PUBLIC_URL` tells Django how the outside world reaches the site (used for generating callback URLs, media paths, etc.). `NGINX_PORT` controls which host port nginx binds to. In dev, they typically match; in production behind a reverse proxy, `PUBLIC_URL` is the external URL and `NGINX_PORT` may differ. +### Using ngrok for GitHub webhooks in local development + +GitHub requires webhook callback URLs to be publicly reachable — it rejects `localhost` URLs. During local development, use [ngrok](https://ngrok.com) to expose your instance: + +```bash +ngrok http 8080 +``` + +Copy the HTTPS URL ngrok gives you (e.g. `https://abc123.ngrok-free.dev`). **Do not** set `PUBLIC_URL` to the ngrok URL — Firebase Auth requires `localhost` for sign-in. Instead, set `GITHUB_HOOK_URL` to the ngrok base URL so only webhook callbacks use it: + +```bash +# In .env.local (or .env): +GITHUB_HOOK_URL=https://abc123.ngrok-free.dev +``` + +Then rebuild and restart: + +```bash +docker compose build && docker compose up -d +``` + +Now you can browse CloudPebble at `http://localhost:8080` (Firebase login works) while GitHub can POST webhook callbacks to the ngrok URL. + ### Test GitHub Repo Sync locally The `GitHub Repo Sync` card changes based on `PUBLIC_URL`, not the browser hostname. diff --git a/cloudpebble/.gitignore b/cloudpebble/.gitignore index 9c17dea..11706eb 100644 --- a/cloudpebble/.gitignore +++ b/cloudpebble/.gitignore @@ -12,3 +12,4 @@ src/ .env/ .idea/ bower_components +node_modules/ diff --git a/cloudpebble/cloudpebble/settings.py b/cloudpebble/cloudpebble/settings.py index 946fd6d..060ed11 100644 --- a/cloudpebble/cloudpebble/settings.py +++ b/cloudpebble/cloudpebble/settings.py @@ -417,7 +417,8 @@ def _redis_db_url(redis_url, db_index): GITHUB_SYNC_CLIENT_ID = _environ.get('GITHUB_SYNC_CLIENT_ID', '') GITHUB_SYNC_CLIENT_SECRET = _environ.get('GITHUB_SYNC_CLIENT_SECRET', '') -GITHUB_HOOK_TEMPLATE = _environ.get('GITHUB_HOOK', PUBLIC_URL.rstrip('/') + '/ide/project/%(project)d/github/push_hook?key=%(key)s') +GITHUB_HOOK_BASE_URL = _environ.get('GITHUB_HOOK_URL', '') +GITHUB_HOOK_TEMPLATE = (GITHUB_HOOK_BASE_URL.rstrip('/') if GITHUB_HOOK_BASE_URL else PUBLIC_URL.rstrip('/')) + '/ide/project/%(project)d/github/push_hook?key=%(key)s' SDK2_PEBBLE_WAF = _environ.get('SDK2_PEBBLE_WAF', '/sdk2/pebble/waf') SDK3_PEBBLE_WAF = _environ.get('SDK3_PEBBLE_WAF', '/sdk3/pebble/waf') diff --git a/cloudpebble/ide/api/git.py b/cloudpebble/ide/api/git.py index 4707b4e..e5becbb 100644 --- a/cloudpebble/ide/api/git.py +++ b/cloudpebble/ide/api/git.py @@ -15,6 +15,19 @@ __author__ = 'katharine' +def _parse_bool(value, default=False): + if value in (True, False): + return value + if isinstance(value, int): + return bool(value) + if isinstance(value, str): + if value in ('1', 'true', 'True'): + return True + if value in ('0', 'false', 'False', ''): + return False + return default + + @login_required @require_POST @json_view @@ -30,7 +43,8 @@ def github_push(request, project_id): @json_view def github_pull(request, project_id): project = get_object_or_404(Project, pk=project_id, owner=request.user) - task = do_github_pull.delay(project.id) + force = _parse_bool(request.POST.get('force', '0')) + task = do_github_pull.delay(project.id, force=force) return {'task_id': task.task_id} @@ -41,8 +55,9 @@ def set_project_repo(request, project_id): project = get_object_or_404(Project, pk=project_id, owner=request.user) repo = request.POST['repo'] branch = request.POST['branch'] - auto_pull = bool(int(request.POST['auto_pull'])) - auto_build = bool(int(request.POST['auto_build'])) + auto_pull = _parse_bool(request.POST['auto_pull']) + auto_build = _parse_bool(request.POST['auto_build']) + hook_force = _parse_bool(request.POST.get('hook_force', '0')) repo = ide.git.url_to_repo(repo) if repo is None: @@ -109,6 +124,7 @@ def set_project_repo(request, project_id): project.github_hook_uuid = None project.github_hook_build = auto_build + project.github_hook_force = hook_force project.save() diff --git a/cloudpebble/ide/api/project.py b/cloudpebble/ide/api/project.py index 2621ee7..feda84d 100644 --- a/cloudpebble/ide/api/project.py +++ b/cloudpebble/ide/api/project.py @@ -454,8 +454,9 @@ def project_info(request, project_id): 'branch': project.github_branch if project.github_branch is not None else None, 'last_sync': str(project.github_last_sync) if project.github_last_sync is not None else None, 'last_commit': project.github_last_commit, - 'auto_build': project.github_hook_build, - 'auto_pull': project.github_hook_uuid is not None + 'auto_build': project.github_hook_build, + 'auto_pull': project.github_hook_uuid is not None, + 'hook_force': project.github_hook_force }, 'supported_platforms': project.supported_platforms, 'has_embeddedjs': project.has_embeddedjs_files, diff --git a/cloudpebble/ide/api/sse.py b/cloudpebble/ide/api/sse.py new file mode 100644 index 0000000..5ceb9de --- /dev/null +++ b/cloudpebble/ide/api/sse.py @@ -0,0 +1,54 @@ +import json +import logging + +from django.contrib.auth.decorators import login_required +from django.http import StreamingHttpResponse +from django.views.decorators.http import require_GET + +from utils.redis_helper import redis_client + +logger = logging.getLogger(__name__) + + +class SSEEventStream: + def __init__(self, project_id): + self.channel = 'project_events:{}'.format(project_id) + self.pubsub = redis_client.pubsub() + self.pubsub.subscribe(self.channel) + + def __iter__(self): + try: + for message in self.pubsub.listen(): + if message['type'] == 'message': + data = message['data'] + if isinstance(data, bytes): + data = data.decode('utf-8') + parsed = json.loads(data) + event_type = parsed.pop('type', 'message') + yield 'event: {}\ndata: {}\n\n'.format(event_type, json.dumps(parsed)) + except GeneratorExit: + pass + finally: + try: + self.pubsub.unsubscribe(self.channel) + self.pubsub.close() + except Exception: + pass + + +@login_required +@require_GET +def project_events(request, project_id): + from ide.models.project import Project + project = Project.objects.filter(pk=project_id, owner=request.user) + if not project.exists(): + from django.http import HttpResponseNotFound + return HttpResponseNotFound() + + response = StreamingHttpResponse( + SSEEventStream(project_id), + content_type='text/event-stream', + ) + response['Cache-Control'] = 'no-cache' + response['X-Accel-Buffering'] = 'no' + return response \ No newline at end of file diff --git a/cloudpebble/ide/migrations/0013_github_hook_force.py b/cloudpebble/ide/migrations/0013_github_hook_force.py new file mode 100644 index 0000000..04ef2e0 --- /dev/null +++ b/cloudpebble/ide/migrations/0013_github_hook_force.py @@ -0,0 +1,16 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('ide', '0012_env_vars'), + ] + + operations = [ + migrations.AddField( + model_name='project', + name='github_hook_force', + field=models.BooleanField(default=False), + ), + ] \ No newline at end of file diff --git a/cloudpebble/ide/models/project.py b/cloudpebble/ide/models/project.py index 04f8afd..354f197 100644 --- a/cloudpebble/ide/models/project.py +++ b/cloudpebble/ide/models/project.py @@ -67,6 +67,7 @@ class Project(IdeModel): github_last_commit = models.CharField(max_length=40, blank=True, null=True) github_hook_uuid = models.CharField(max_length=36, blank=True, null=True) github_hook_build = models.BooleanField(default=False) + github_hook_force = models.BooleanField(default=False) project_dependencies = models.ManyToManyField("Project", db_table='cloudpebble_project_dependencies') diff --git a/cloudpebble/ide/static/ide/css/ide.css b/cloudpebble/ide/static/ide/css/ide.css index 9f35bc8..1aeac74 100644 --- a/cloudpebble/ide/static/ide/css/ide.css +++ b/cloudpebble/ide/static/ide/css/ide.css @@ -1616,4 +1616,14 @@ label .text-icon { } .spinner-light { background-image: url("/static/ide/img/spinner.gif"); +} + +.sidebar-pending { + display: inline-block; + font-size: 11px; + color: #ff4700; + margin-left: 6px; + font-family: sans-serif; + font-weight: bold; + vertical-align: middle; } \ No newline at end of file diff --git a/cloudpebble/ide/static/ide/js/__tests__/sidebar-refresh.test.js b/cloudpebble/ide/static/ide/js/__tests__/sidebar-refresh.test.js new file mode 100644 index 0000000..94d6b61 --- /dev/null +++ b/cloudpebble/ide/static/ide/js/__tests__/sidebar-refresh.test.js @@ -0,0 +1,217 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { readFileSync } from 'fs'; +import { resolve } from 'path'; + +function makeJqueryMock() { + var elements = {}; + var $ = function(selector) { + if (typeof selector === 'function') { + selector(); + return $; + } + var el = elements[selector] || { + empty: vi.fn(() => el), + append: vi.fn(() => el), + remove: vi.fn(() => el), + data: vi.fn(() => undefined), + find: vi.fn(() => $), + children: vi.fn(() => []), + addClass: vi.fn(() => el), + removeClass: vi.fn(() => el), + detach: vi.fn(() => el), + text: vi.fn((v) => v !== undefined ? (el._text = v, el) : (el._text || '')), + attr: vi.fn(() => el), + click: vi.fn(() => el), + siblings: vi.fn(() => $), + slideToggle: vi.fn(), + closest: vi.fn(() => $), + length: 0 + }; + elements[selector] = el; + return el; + }; + $.Deferred = vi.fn(() => { + var callbacks = { done: [], fail: [], always: [] }; + var promise = { + done: vi.fn((cb) => { callbacks.done.push(cb); return promise; }), + fail: vi.fn((cb) => { callbacks.fail.push(cb); return promise; }), + always: vi.fn((cb) => { callbacks.always.push(cb); return promise; }), + then: vi.fn((successCb, failCb) => { + if (successCb) callbacks.done.push(successCb); + if (failCb) callbacks.fail.push(failCb); + return promise; + }), + promise: vi.fn(() => promise) + }; + var deferred = { + resolve: vi.fn((...args) => { + callbacks.done.forEach(cb => cb(...args)); + callbacks.always.forEach(cb => cb()); + }), + reject: vi.fn((...args) => { + callbacks.fail.forEach(cb => cb(...args)); + callbacks.always.forEach(cb => cb()); + }), + promise: vi.fn(() => promise), + then: promise.then, + done: promise.done, + fail: promise.fail, + always: promise.always + }; + return deferred; + }); + $.each = vi.fn((arr, fn) => { + if (Array.isArray(arr)) arr.forEach((v, i) => fn(i, v)); + else if (typeof arr === 'object') Object.keys(arr).forEach(k => fn(k, arr[k])); + }); + $._elements = elements; + return $; +} + +function makeCloudPebble() { + return { + Editor: { + GetUnsavedFiles: vi.fn(() => 3), + Add: vi.fn(), + ReloadActive: vi.fn(() => Promise.resolve()) + }, + Resources: { + Add: vi.fn(), + AddAlloyAsset: vi.fn() + }, + ProjectInfo: { type: 'native' }, + TargetNames: { app: 'App', pkjs: 'PKJS' } + }; +} + +function loadSidebarModule(CloudPebble, $, Ajax) { + global.PROJECT_ID = 1; + global.CloudPebble = CloudPebble; + global.$ = $; + global.jQuery = $; + global.Ajax = Ajax; + global._ = { each: vi.fn((arr, fn) => arr.forEach(fn)) }; + + var sidebarPath = resolve(__dirname, '..', 'sidebar.js'); + var code = readFileSync(sidebarPath, 'utf8'); + var fn = new Function(code); + fn(); + + return CloudPebble.Sidebar; +} + +describe('Sidebar.Refresh', () => { + let CloudPebble; + let Sidebar; + let originalGetUnsavedFiles; + let $; + let deferred; + + beforeEach(() => { + $ = makeJqueryMock(); + CloudPebble = makeCloudPebble(); + originalGetUnsavedFiles = CloudPebble.Editor.GetUnsavedFiles; + deferred = $.Deferred(); + var Ajax = { + Get: vi.fn(() => deferred.promise()) + }; + Sidebar = loadSidebarModule(CloudPebble, $, Ajax); + }); + + it('temporarily overrides GetUnsavedFiles to return 0 during refresh', () => { + Sidebar.Refresh(); + expect(CloudPebble.Editor.GetUnsavedFiles()).toBe(0); + }); + + it('restores GetUnsavedFiles after successful ajax response', () => { + Sidebar.Refresh(); + expect(CloudPebble.Editor.GetUnsavedFiles()).toBe(0); + + deferred.resolve({ + type: 'native', + source_files: [], + resources: [] + }); + + expect(CloudPebble.Editor.GetUnsavedFiles).toBe(originalGetUnsavedFiles); + }); + + it('restores GetUnsavedFiles even when ajax request fails', () => { + Sidebar.Refresh(); + expect(CloudPebble.Editor.GetUnsavedFiles()).toBe(0); + + deferred.reject(); + + expect(CloudPebble.Editor.GetUnsavedFiles).toBe(originalGetUnsavedFiles); + }); + + it('restores GetUnsavedFiles when done callback throws', () => { + CloudPebble.Editor.Add = vi.fn(() => { throw new Error('boom'); }); + + Sidebar.Refresh(); + try { + deferred.resolve({ + type: 'native', + source_files: [{ target: 'app', name: 'main.c' }], + resources: [] + }); + } catch (e) {} + + expect(CloudPebble.Editor.GetUnsavedFiles).toBe(originalGetUnsavedFiles); + }); + + it('updates ProjectInfo from ajax response', () => { + var newData = { type: 'pebblejs', source_files: [], resources: [] }; + Sidebar.Refresh(); + deferred.resolve(newData); + expect(CloudPebble.ProjectInfo).toBe(newData); + }); + + it('adds source files from response', () => { + Sidebar.Refresh(); + deferred.resolve({ + type: 'native', + source_files: [{ target: 'app', name: 'main.c' }], + resources: [] + }); + expect(CloudPebble.Editor.Add).toHaveBeenCalled(); + }); + + it('adds resources from response', () => { + Sidebar.Refresh(); + deferred.resolve({ + type: 'native', + source_files: [], + resources: [{ id: 1, file_name: 'icon.png' }] + }); + expect(CloudPebble.Resources.Add).toHaveBeenCalledWith({ id: 1, file_name: 'icon.png' }); + }); + + it('calls Editor.ReloadActive after a successful refresh', () => { + Sidebar.Refresh(); + deferred.resolve({ + type: 'native', + source_files: [], + resources: [] + }); + expect(CloudPebble.Editor.ReloadActive).toHaveBeenCalled(); + }); + + it('does not call Editor.ReloadActive when the ajax request fails', () => { + Sidebar.Refresh(); + deferred.reject(); + expect(CloudPebble.Editor.ReloadActive).not.toHaveBeenCalled(); + }); + + it('does not throw when Editor.ReloadActive is not provided', () => { + delete CloudPebble.Editor.ReloadActive; + Sidebar.Refresh(); + expect(() => { + deferred.resolve({ + type: 'native', + source_files: [], + resources: [] + }); + }).not.toThrow(); + }); +}); \ No newline at end of file diff --git a/cloudpebble/ide/static/ide/js/__tests__/sse-events.test.js b/cloudpebble/ide/static/ide/js/__tests__/sse-events.test.js new file mode 100644 index 0000000..c29f5bc --- /dev/null +++ b/cloudpebble/ide/static/ide/js/__tests__/sse-events.test.js @@ -0,0 +1,110 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { readFileSync } from 'fs'; +import { resolve } from 'path'; + +function makeCloudPebble() { + return { + Sidebar: { + ShowPending: vi.fn(), + HidePending: vi.fn() + }, + GitHub: { + OnPullStart: vi.fn(), + OnPullComplete: vi.fn(), + OnPullFailed: vi.fn() + }, + Compile: { + OnBuildStart: vi.fn(), + OnBuildComplete: vi.fn() + } + }; +} + +function loadSseModule(CloudPebble) { + global.PROJECT_ID = 1; + global.CloudPebble = CloudPebble; + global.EventSource = vi.fn().mockImplementation(() => ({ + addEventListener: vi.fn(), + close: vi.fn(), + readyState: 0 + })); + + const ssePath = resolve(__dirname, '..', 'sse.js'); + const code = readFileSync(ssePath, 'utf8'); + const fn = new Function(code); + fn(); + + return CloudPebble.Events; +} + +describe('sse event handlers', () => { + let CloudPebble; + let events; + + beforeEach(() => { + CloudPebble = makeCloudPebble(); + events = loadSseModule(CloudPebble); + vi.clearAllMocks(); + }); + + describe('handlePullStart', () => { + it('shows pending pill with Pulling text and calls OnPullStart', () => { + events.handlePullStart(); + expect(CloudPebble.Sidebar.ShowPending).toHaveBeenCalledWith('github', 'Pulling'); + expect(CloudPebble.GitHub.OnPullStart).toHaveBeenCalled(); + }); + }); + + describe('handlePullComplete', () => { + it('hides pending pill and calls OnPullComplete with parsed data', () => { + const event = { data: JSON.stringify({ github_last_commit: 'abc123' }) }; + events.handlePullComplete(event); + expect(CloudPebble.Sidebar.HidePending).toHaveBeenCalledWith('github'); + expect(CloudPebble.GitHub.OnPullComplete).toHaveBeenCalledWith({ github_last_commit: 'abc123' }); + }); + + it('handles pull_complete with empty data', () => { + const event = { data: JSON.stringify({}) }; + events.handlePullComplete(event); + expect(CloudPebble.GitHub.OnPullComplete).toHaveBeenCalledWith({}); + }); + }); + + describe('handlePullFailed', () => { + it('hides pending pill and calls OnPullFailed', () => { + events.handlePullFailed(); + expect(CloudPebble.Sidebar.HidePending).toHaveBeenCalledWith('github'); + expect(CloudPebble.GitHub.OnPullFailed).toHaveBeenCalled(); + }); + }); + + describe('handleBuildStart', () => { + it('shows pending pill with Building text and calls OnBuildStart with build_id', () => { + const event = { data: JSON.stringify({ build_id: 42 }) }; + events.handleBuildStart(event); + expect(CloudPebble.Sidebar.ShowPending).toHaveBeenCalledWith('compile', 'Building'); + expect(CloudPebble.Compile.OnBuildStart).toHaveBeenCalledWith(42); + }); + + it('parses build_id as integer from JSON data', () => { + const event = { data: JSON.stringify({ build_id: 99 }) }; + events.handleBuildStart(event); + expect(CloudPebble.Compile.OnBuildStart).toHaveBeenCalledWith(99); + }); + }); + + describe('handleBuildComplete', () => { + it('hides pending pill and calls OnBuildComplete with build_id and state', () => { + const event = { data: JSON.stringify({ build_id: 42, state: 'succeeded' }) }; + events.handleBuildComplete(event); + expect(CloudPebble.Sidebar.HidePending).toHaveBeenCalledWith('compile'); + expect(CloudPebble.Compile.OnBuildComplete).toHaveBeenCalledWith(42, 'succeeded'); + }); + + it('passes failed state correctly', () => { + const event = { data: JSON.stringify({ build_id: 7, state: 'failed' }) }; + events.handleBuildComplete(event); + expect(CloudPebble.Compile.OnBuildComplete).toHaveBeenCalledWith(7, 'failed'); + }); + }); +}); \ No newline at end of file diff --git a/cloudpebble/ide/static/ide/js/cloudpebble.js b/cloudpebble/ide/static/ide/js/cloudpebble.js index 621617b..f102e16 100644 --- a/cloudpebble/ide/static/ide/js/cloudpebble.js +++ b/cloudpebble/ide/static/ide/js/cloudpebble.js @@ -93,6 +93,7 @@ CloudPebble.Init = function() { CloudPebble.Publish.Init(); } CloudPebble.FuzzyPrompt.Init(); + CloudPebble.Events.Init(); CloudPebble.ProgressBar.Hide(); // Add source files. diff --git a/cloudpebble/ide/static/ide/js/compile.js b/cloudpebble/ide/static/ide/js/compile.js index d44f98e..54d3f25 100644 --- a/cloudpebble/ide/static/ide/js/compile.js +++ b/cloudpebble/ide/static/ide/js/compile.js @@ -947,6 +947,20 @@ var show_clear_logs_prompt = function() { var opts = options || {}; var kind = opts.kind || CloudPebble.Compile.GetPlatformForInstall(); return install_on_watch(kind, opts.build); + }, + OnBuildStart: function(buildId) { + mRunningBuild = true; + if (pane) { + var tempBuild = {started: (new Date()).toISOString(), finished: null, state: 1, uuid: null, id: buildId, size: {total: null, binary: null, resources: null}}; + update_last_build(pane, tempBuild); + pane.find('#run-build-table').prepend(build_history_row(tempBuild)); + } + }, + OnBuildComplete: function(buildId, state) { + mRunningBuild = false; + if (pane) { + update_build_history(pane); + } } }; })(); diff --git a/cloudpebble/ide/static/ide/js/editor.js b/cloudpebble/ide/static/ide/js/editor.js index 12c7a68..da5d185 100644 --- a/cloudpebble/ide/static/ide/js/editor.js +++ b/cloudpebble/ide/static/ide/js/editor.js @@ -519,12 +519,12 @@ CloudPebble.Editor = (function() { var check_safe = function() { return Ajax.Get('/ide/project/' + PROJECT_ID + '/source/' + file.id + '/is_safe?modified=' + file.lastModified).then(function(data) { if(!data.safe) { - if(was_clean) { + if(code_mirror.was_clean) { code_mirror.setOption('readOnly', true); return CloudPebble.Get('/ide/project/' + PROJECT_ID + '/source/' + file.id + '/load', function(data) { code_mirror.setValue(data.source); file.lastModified = data.modified; - was_clean = true; // this will get reset to false by setValue. + code_mirror.was_clean = true; // this will get reset to false by setValue. }).finally(function() { code_mirror.setOption('readOnly', false); }); @@ -552,24 +552,24 @@ CloudPebble.Editor = (function() { } }, onDestroy: function() { - if(!was_clean) { + if(!code_mirror.was_clean) { --unsaved_files; } delete open_codemirrors[file.id]; } }); - var was_clean = true; + code_mirror.was_clean = true; code_mirror.on('change', function() { - if(was_clean) { + if(code_mirror.was_clean) { CloudPebble.Sidebar.SetIcon('source-' + file.id, 'edit'); - was_clean = false; + code_mirror.was_clean = false; ++unsaved_files; } }); var mark_clean = function() { - was_clean = true; + code_mirror.was_clean = true; --unsaved_files; CloudPebble.Sidebar.ClearIcon('source-' + file.id); }; @@ -1284,6 +1284,18 @@ CloudPebble.Editor = (function() { } } + var get_active_source_file_id = function() { + var pane_id = $('#main-pane').data('pane-id'); + if (!pane_id || pane_id.indexOf('source-') !== 0) { + return null; + } + var file_id = parseInt(pane_id.substring('source-'.length), 10); + if (isNaN(file_id)) { + return null; + } + return file_id; + }; + return { Create: function() { create_source_file(); @@ -1311,6 +1323,37 @@ CloudPebble.Editor = (function() { }, RenameFile: function(file, new_name) { return rename_file(file, new_name) + }, + /** + * Re-fetch the source of the currently-active editor from the server and + * replace its buffer. No-op if the active pane isn't a source file, if + * the editor is currently detached, or if the buffer has unsaved local + * edits (we never clobber user work). + * + * Called from Sidebar.Refresh after a GitHub pull, so the user doesn't + * end up staring at a stale buffer for a file that just changed on + * the server. + */ + ReloadActive: function() { + var file_id = get_active_source_file_id(); + if (file_id === null) { + return Promise.resolve(); + } + var code_mirror = open_codemirrors[file_id]; + if (!code_mirror) { + return Promise.resolve(); + } + if (!code_mirror.was_clean) { + return Promise.resolve(); + } + return Ajax.Get('/ide/project/' + PROJECT_ID + '/source/' + file_id + '/load').then(function(data) { + code_mirror.setValue(data.source); + code_mirror.was_clean = true; + var file = project_source_files[code_mirror.file_path]; + if (file) { + file.lastModified = data.modified; + } + }); } }; })(); diff --git a/cloudpebble/ide/static/ide/js/github.js b/cloudpebble/ide/static/ide/js/github.js index 25d09f7..30bd341 100644 --- a/cloudpebble/ide/static/ide/js/github.js +++ b/cloudpebble/ide/static/ide/js/github.js @@ -36,6 +36,22 @@ CloudPebble.GitHub = (function() { var enable_needy = function() { pane.find('.github-actions').find('input, button').removeAttr('disabled'); }; + var update_pull_mode_ui = function() { + var auto_pull = pane.find('#github-repo-hook').val() == '1'; + var force_checkbox = pane.find('#github-repo-hook-force'); + var help_text = pane.find('#github-repo-hook-help'); + if (auto_pull) { + force_checkbox.removeAttr('disabled'); + if (force_checkbox.is(':checked')) { + help_text.html(gettext("Auto-pull will wipe and re-import all files from GitHub every time you push. This is slower but more thorough than the default delta sync.")); + } else { + help_text.html(gettext("Auto-pull will sync only the changed files from GitHub every time you push. This is fast but check the box above for a full re-import if you encounter issues.")); + } + } else { + force_checkbox.attr('disabled', 'disabled'); + help_text.html(gettext("You will need to pull from GitHub manually using the button below. Auto-pull is disabled.")); + } + }; pane.find('#github-repo-form').submit(function(e) { e.preventDefault(); @@ -44,6 +60,7 @@ CloudPebble.GitHub = (function() { var repo_branch = pane.find('#github-branch').val(); var auto_pull = pane.find('#github-repo-hook').val() == '1'; var auto_build = pane.find('#github-repo-build').val() == '1'; + var hook_force = pane.find('#github-repo-hook-force').is(':checked'); if(repo_branch == null || repo_branch.length == 0) { repo_branch = "master"; @@ -51,7 +68,7 @@ CloudPebble.GitHub = (function() { if((new_repo === CloudPebble.ProjectInfo.github.repo || !new_repo && !CloudPebble.ProjectInfo.github.repo) && (repo_branch === CloudPebble.ProjectInfo.github.branch || !repo_branch && !CloudPebble.ProjectInfo.github.branch) && - auto_pull === CloudPebble.ProjectInfo.github.auto_pull && auto_build === CloudPebble.ProjectInfo.github.auto_build) { + auto_pull === CloudPebble.ProjectInfo.github.auto_pull && auto_build === CloudPebble.ProjectInfo.github.auto_build && hook_force === CloudPebble.ProjectInfo.github.hook_force) { show_alert('success', "Updated repo (nothing changed)."); return; } @@ -60,6 +77,7 @@ CloudPebble.GitHub = (function() { repo: new_repo, auto_pull: auto_pull ? '1' : '0', auto_build: auto_build ? '1' : '0', + hook_force: hook_force ? '1' : '0', branch: repo_branch }; Ajax.Post('/ide/project/' + PROJECT_ID + '/github/repo', data).then(function(data) { @@ -70,6 +88,7 @@ CloudPebble.GitHub = (function() { CloudPebble.ProjectInfo.github.branch = repo_branch; CloudPebble.ProjectInfo.github.auto_pull = auto_pull; CloudPebble.ProjectInfo.github.auto_build = auto_build; + CloudPebble.ProjectInfo.github.hook_force = hook_force; return; } if(!data.exists) { @@ -104,6 +123,14 @@ CloudPebble.GitHub = (function() { } pane.find('#github-repo-hook').val(CloudPebble.ProjectInfo.github.auto_pull ? '1' : '0'); pane.find('#github-repo-build').val(CloudPebble.ProjectInfo.github.auto_build ? '1' : '0'); + pane.find('#github-repo-hook-force').prop('checked', CloudPebble.ProjectInfo.github.hook_force); + pane.find('#github-repo-hook').on('change', update_pull_mode_ui); + pane.find('#github-repo-hook-force').on('change', update_pull_mode_ui); + update_pull_mode_ui(); + var lastSync = CloudPebble.ProjectInfo.github.last_sync; + if(lastSync) { + pane.find('#github-last-sync').text(interpolate(gettext('Last synced: %s'), [lastSync])); + } var prompt = $('#github-new-repo-prompt'); prompt.find('form').submit(function(e) { @@ -137,6 +164,12 @@ CloudPebble.GitHub = (function() { var prompt = $('#github-pull-prompt').modal(); prompt.find(".running").addClass('hide'); prompt.find(".close, .dire-warning, .modal-footer").removeClass("hide"); + prompt.find('#github-pull-force').prop('checked', false); + prompt.find('#github-pull-force-warning').addClass('hide'); + }); + + $(document).on('change', '#github-pull-force', function() { + $('#github-pull-force-warning').toggleClass('hide', !$(this).is(':checked')); }); var poll_commit_status = function(task_id) { @@ -156,13 +189,13 @@ CloudPebble.GitHub = (function() { return Ajax.PollTask(task_id).then(function(result) { if(result) { show_alert('success', gettext("Pulled successfully.")); - alert(gettext("Pull completed successfully.")); - // *NASTY HACK: Make sure it doesn't think we have unsaved files, thereby - // preventing page reload. - CloudPebble.Editor.GetUnsavedFiles = function() { return 0; }; - window.location.reload(true); } else { - show_alert('success', gettext("Pull completed: Nothing to pull.")); + var lastSync = CloudPebble.ProjectInfo.github.last_sync; + if(lastSync) { + show_alert('success', interpolate(gettext("Already up to date (last synced %s)."), [lastSync])); + } else { + show_alert('success', gettext("Pull completed: Nothing to pull.")); + } } }); }; @@ -196,15 +229,14 @@ CloudPebble.GitHub = (function() { $('#github-pull-prompt-confirm').click(function() { disable_all(); var prompt = $('#github-pull-prompt'); - prompt.find(".close, .dire-warning, .modal-footer").addClass("hide"); - prompt.find(".running").removeClass('hide'); - Ajax.Post('/ide/project/' + PROJECT_ID + '/github/pull').then(function(data) { + prompt.modal('hide'); + var forcePull = prompt.find('#github-pull-force').is(':checked') ? '1' : '0'; + Ajax.Post('/ide/project/' + PROJECT_ID + '/github/pull', {force: forcePull}).then(function(data) { return poll_pull_status(data.task_id); }).catch(function(error) { show_alert('error', interpolate(gettext("Pull failed: %s"), [error.message])); }).finally(function() { enable_all(); - prompt.modal('hide'); }); ga('send', 'event', 'github', 'pull'); }); @@ -221,6 +253,20 @@ CloudPebble.GitHub = (function() { }, Show: function() { show_github_pane(); + }, + OnPullStart: function() { + $('#github-push-btn, #github-pull-btn').attr('disabled', 'disabled'); + $('#github-last-sync').text(gettext('Pulling from GitHub...')); + }, + OnPullComplete: function(data) { + $('#github-push-btn, #github-pull-btn').removeAttr('disabled'); + var lastSync = data && data.github_last_sync ? data.github_last_sync : ''; + $('#github-last-sync').text(lastSync ? interpolate(gettext('Last synced: %s'), [lastSync]) : gettext('Pull completed.')); + CloudPebble.Sidebar.Refresh(); + }, + OnPullFailed: function() { + $('#github-push-btn, #github-pull-btn').removeAttr('disabled'); + $('#github-last-sync').text(gettext('Pull failed.')); } }; })(); diff --git a/cloudpebble/ide/static/ide/js/sidebar.js b/cloudpebble/ide/static/ide/js/sidebar.js index f4045d9..94cfbb3 100644 --- a/cloudpebble/ide/static/ide/js/sidebar.js +++ b/cloudpebble/ide/static/ide/js/sidebar.js @@ -243,8 +243,30 @@ CloudPebble.Sidebar = (function() { a.find('i').removeClass().addClass('icon-' + icon); }, + ShowPending: function(pane_id, text) { + if (this._pendingTimers && this._pendingTimers[pane_id]) { + clearInterval(this._pendingTimers[pane_id]); + } + this.ClearIcon(pane_id); + var a = $('#sidebar-pane-' + pane_id).find('a'); + var span = $('').text(text); + a.append(span); + var dots = 1; + this._pendingTimers = this._pendingTimers || {}; + this._pendingTimers[pane_id] = setInterval(function() { + span.text(text + '.'.repeat(dots)); + dots = (dots % 3) + 1; + }, 1000); + }, + HidePending: function(pane_id) { + if (this._pendingTimers && this._pendingTimers[pane_id]) { + clearInterval(this._pendingTimers[pane_id]); + delete this._pendingTimers[pane_id]; + } + this.ClearIcon(pane_id); + }, ClearIcon: function(pane_id) { - $('#sidebar-pane-' + pane_id).find('a > i').remove(); + $('#sidebar-pane-' + pane_id).find('a > i, a > .sidebar-pending').remove(); }, Init: function() { $('#sidebar-pane-new-resource').click(CloudPebble.Resources.Create); @@ -293,6 +315,40 @@ CloudPebble.Sidebar = (function() { if(type != 'alloy') { $('.alloy-only').hide(); } + }, + Refresh: function() { + var saved_GetUnsavedFiles = CloudPebble.Editor.GetUnsavedFiles; + CloudPebble.Editor.GetUnsavedFiles = function() { return 0; }; + $('#sidebar-sources').empty(); + $('#sidebar-resources').empty(); + create_initial_sections(CloudPebble.ProjectInfo.type); + Ajax.Get('/ide/project/' + PROJECT_ID + '/info').done(function(data) { + try { + CloudPebble.ProjectInfo = data; + var is_alloy = data.type === 'alloy'; + $.each(data.source_files, function(index, value) { + if (is_alloy && value.target === 'embeddedjs' && value.is_binary) { + if (CloudPebble.Resources && _.isFunction(CloudPebble.Resources.AddAlloyAsset)) { + CloudPebble.Resources.AddAlloyAsset(value); + } + return; + } + CloudPebble.Editor.Add(value); + }); + $.each(data.resources, function(index, value) { + CloudPebble.Resources.Add(value); + }); + } finally { + CloudPebble.Editor.GetUnsavedFiles = saved_GetUnsavedFiles; + } + // Re-load the buffer for the currently-open file, if any, so + // the user isn't staring at stale content after a pull. + if (typeof CloudPebble.Editor.ReloadActive === 'function') { + CloudPebble.Editor.ReloadActive(); + } + }).fail(function() { + CloudPebble.Editor.GetUnsavedFiles = saved_GetUnsavedFiles; + }); } }; })(); \ No newline at end of file diff --git a/cloudpebble/ide/static/ide/js/sse.js b/cloudpebble/ide/static/ide/js/sse.js new file mode 100644 index 0000000..570f995 --- /dev/null +++ b/cloudpebble/ide/static/ide/js/sse.js @@ -0,0 +1,67 @@ +CloudPebble.Events = (function() { + var source = null; + var reconnectTimer = null; + var RECONNECT_DELAY = 3000; + + var handlers = { + handlePullStart: function() { + CloudPebble.Sidebar.ShowPending('github', 'Pulling'); + CloudPebble.GitHub.OnPullStart(); + }, + handlePullComplete: function(e) { + CloudPebble.Sidebar.HidePending('github'); + CloudPebble.GitHub.OnPullComplete(JSON.parse(e.data)); + }, + handlePullFailed: function() { + CloudPebble.Sidebar.HidePending('github'); + CloudPebble.GitHub.OnPullFailed(); + }, + handleBuildStart: function(e) { + var data = JSON.parse(e.data); + CloudPebble.Sidebar.ShowPending('compile', 'Building'); + CloudPebble.Compile.OnBuildStart(data.build_id); + }, + handleBuildComplete: function(e) { + var data = JSON.parse(e.data); + CloudPebble.Sidebar.HidePending('compile'); + CloudPebble.Compile.OnBuildComplete(data.build_id, data.state); + } + }; + + var connect = function() { + if (source) { + source.close(); + } + + source = new EventSource('/ide/project/' + PROJECT_ID + '/events'); + + source.addEventListener('pull_start', handlers.handlePullStart); + source.addEventListener('pull_complete', handlers.handlePullComplete); + source.addEventListener('pull_failed', handlers.handlePullFailed); + source.addEventListener('build_start', handlers.handleBuildStart); + source.addEventListener('build_complete', handlers.handleBuildComplete); + + source.onerror = function() { + if (source.readyState === EventSource.CLOSED) { + if (reconnectTimer) clearTimeout(reconnectTimer); + reconnectTimer = setTimeout(connect, RECONNECT_DELAY); + } + }; + }; + + return { + Init: function() { + connect(); + }, + Close: function() { + if (reconnectTimer) clearTimeout(reconnectTimer); + if (source) source.close(); + source = null; + }, + handlePullStart: handlers.handlePullStart, + handlePullComplete: handlers.handlePullComplete, + handlePullFailed: handlers.handlePullFailed, + handleBuildStart: handlers.handleBuildStart, + handleBuildComplete: handlers.handleBuildComplete + }; +})(); \ No newline at end of file diff --git a/cloudpebble/ide/tasks/archive.py b/cloudpebble/ide/tasks/archive.py index 0a8ad6f..f57caff 100644 --- a/cloudpebble/ide/tasks/archive.py +++ b/cloudpebble/ide/tasks/archive.py @@ -145,7 +145,7 @@ def ends_with_any(s, options): @shared_task(acks_late=True) -def do_import_archive(project_id, archive, delete_project=False): +def do_import_archive(project_id, archive, delete_project=False, wipe_existing=False): project = Project.objects.get(pk=project_id) try: with tempfile.NamedTemporaryFile(suffix='.zip') as archive_file: @@ -201,6 +201,9 @@ def make_valid_filename(zip_entry): filtered_contents.append((filename, entry)) with transaction.atomic(): + if wipe_existing: + project.source_files.all().delete() + project.resources.all().delete() # We have a resource map! We can now try importing things from it. project_options, media_map, dependencies = load_manifest_dict(manifest_dict, manifest_kind) diff --git a/cloudpebble/ide/tasks/build.py b/cloudpebble/ide/tasks/build.py index 718c623..89617b6 100644 --- a/cloudpebble/ide/tasks/build.py +++ b/cloudpebble/ide/tasks/build.py @@ -16,6 +16,7 @@ from ide.utils.crypto import decrypt_value from ide.utils.sdk.project_assembly import assemble_project from utils.td_helper import send_td_event +from utils.events import publish_event __author__ = 'katharine' @@ -179,6 +180,9 @@ def run_compile(build_result): build_result.state = BuildResult.STATE_SUCCEEDED if success else BuildResult.STATE_FAILED build_result.finished = now() build_result.save() + publish_event(build_result.project_id, 'build_complete', + build_id=build_result.id, + state='succeeded' if success else 'failed') data = { 'data': { @@ -203,5 +207,7 @@ def run_compile(build_result): except: pass build_result.save() + publish_event(build_result.project_id, 'build_complete', + build_id=build_result.id, state='failed') finally: shutil.rmtree(base_dir) diff --git a/cloudpebble/ide/tasks/git.py b/cloudpebble/ide/tasks/git.py index 7cd76ff..ad3822d 100644 --- a/cloudpebble/ide/tasks/git.py +++ b/cloudpebble/ide/tasks/git.py @@ -1,4 +1,5 @@ import base64 +import io from urllib.request import urlopen, Request from urllib.error import URLError, HTTPError import json @@ -6,18 +7,22 @@ import logging from celery import shared_task +from django.db import transaction from django.utils.timezone import now from github.GithubObject import NotSet from github import Github, GithubException, InputGitTreeElement from ide.git import git_auth_check, get_github from ide.models.build import BuildResult +from ide.models.files import SourceFile, ResourceFile, ResourceVariant, ResourceIdentifier from ide.models.project import Project from ide.tasks import do_import_archive, run_compile +from ide.tasks.archive import get_filename_variant from ide.utils.git import git_sha, git_blob from ide.utils.project import find_project_root_and_manifest, BaseProjectItem, InvalidProjectArchiveException -from ide.utils.sdk import generate_manifest_dict, generate_manifest, generate_wscript_file, manifest_name_for_project +from ide.utils.sdk import generate_manifest_dict, generate_manifest, generate_wscript_file, manifest_name_for_project, load_manifest_dict from utils.td_helper import send_td_event +from utils.events import publish_event __author__ = 'katharine' @@ -313,74 +318,412 @@ def path(self): return self.git_item.path +PEBBLEJS_BUILTIN_RESOURCES = frozenset({ + 'MONO_FONT_14', 'IMAGE_MENU_ICON', 'IMAGE_LOGO_SPLASH', 'IMAGE_TILE_SPLASH', +}) + + +def validate_resources_against_tree(paths_notags, manifest, project, root=''): + """Validate that all resources referenced in the manifest exist in the tree. + + Given a set of tag-stripped paths from a git tree and a parsed manifest dict, + checks that every resource file listed in the manifest is present in the tree. + Skips built-in Pebble.js resources that don't need to be in the repo. + + Returns the manifest's media list for further processing. + + Raises Exception if a required resource is missing. + """ + resource_root = ((root + '/' if root else '') + project.resources_path).rstrip('/') + '/' + pebble = manifest.get('pebble', manifest) + manifest_resources = pebble.get('resources', {}).get('media', []) + project_type = pebble.get('projectType', 'native') + + for resource in manifest_resources: + path = resource_root + resource['file'] + if project_type == 'pebblejs' and resource['name'] in PEBBLEJS_BUILTIN_RESOURCES: + continue + if path not in paths_notags: + raise Exception("Resource %s not found in repo." % path) + + return manifest_resources + + +def parse_manifest_from_tree(tree_items, project): + """Find and parse the manifest from a git tree, returning (root, manifest_dict). + + Takes a list of BaseProjectItem instances and a Project. Returns a tuple of + (project_root_path, manifest_dict). Raises ValueError or + InvalidProjectArchiveException if the tree has no valid manifest. + """ + root, manifest_item = find_project_root_and_manifest(tree_items) + manifest = json.loads(manifest_item.read()) + return root, manifest + + @git_auth_check -def github_pull(user, project): +def github_pull(user, project, force=False): g = get_github(user) repo_name = project.github_repo if repo_name is None: raise Exception("No GitHub repo defined.") repo = g.get_repo(repo_name) - # If somehow we don't have a branch set, this will use the default branch branch_name = project.github_branch or repo.default_branch try: branch = repo.get_branch(branch_name) except GithubException: raise Exception("Unable to get the branch.") - if project.github_last_commit == branch.commit.sha: + new_commit_sha = branch.commit.sha + + if project.github_last_commit == new_commit_sha: # Nothing to do. return False + # Use full wipe-and-replace for force pulls or when we have no previous commit + if force or project.github_last_commit is None: + return _github_pull_full(user, project, repo, branch) + + # Try incremental delta sync + try: + return _github_pull_delta(user, project, repo, new_commit_sha) + except Exception as e: + logger.warning("Delta sync failed (%s), falling back to full pull", e) + return _github_pull_full(user, project, repo, branch) + + +def _github_pull_full(user, project, repo, branch): + """Full wipe-and-replace pull: downloads entire zip and re-imports everything.""" + branch_name = project.github_branch or repo.default_branch commit = repo.get_git_commit(branch.commit.sha) tree = repo.get_git_tree(commit.tree.sha, recursive=True) - paths = {x.path: x for x in tree.tree} - paths_notags = {get_root_path(x) for x in paths} + paths_notags = {get_root_path(x.path) for x in tree.tree} - # First try finding the resource map so we don't fail out part-done later. try: - root, manifest_item = find_project_root_and_manifest([GitProjectItem(repo, x) for x in tree.tree]) + root, manifest = parse_manifest_from_tree( + [GitProjectItem(repo, x) for x in tree.tree], project) except ValueError as e: raise ValueError("In manifest file: %s" % str(e)) - resource_root = root + project.resources_path + '/' - manifest = json.loads(manifest_item.read()) - media = manifest.get('resources', {}).get('media', []) - project_type = manifest.get('projectType', 'native') + validate_resources_against_tree(paths_notags, manifest, project, root) - for resource in media: - path = resource_root + resource['file'] - if project_type == 'pebblejs' and resource['name'] in { - 'MONO_FONT_14', 'IMAGE_MENU_ICON', 'IMAGE_LOGO_SPLASH', 'IMAGE_TILE_SPLASH'}: - continue - if path not in paths_notags: - raise Exception("Resource %s not found in repo." % path) - - # Now we grab the zip. + # Start the zip download in parallel with validation. zip_url = repo.get_archive_link('zipball', branch_name) u = urlopen(zip_url) - # And wipe the project! - # TODO: transaction support for file contents would be nice... - project.source_files.all().delete() - project.resources.all().delete() - - # This must happen before do_import_archive or we'll stamp on its results. - project.github_last_commit = branch.commit.sha - project.github_last_sync = now() - project.save() + import_result = do_import_archive(project.id, u.read(), wipe_existing=True) - import_result = do_import_archive(project.id, u.read()) + # Stamp the new SHA inside the same transaction as the archive import so + # the project never ends up with new files but the old github_last_commit + # (which would cause the next pull to re-apply the same archive). + with transaction.atomic(): + project.github_last_commit = branch.commit.sha + project.github_last_sync = now() + project.save() send_td_event('cloudpebble_github_pull', data={ - 'data': { - 'repo': project.github_repo - } + 'data': {'repo': project.github_repo} }, user=user) return import_result +def _github_pull_delta(user, project, repo, new_commit_sha): + """Incremental pull: only processes files that changed between commits.""" + comparison = repo.compare(project.github_last_commit, new_commit_sha) + + if comparison.ahead_by == 0: + with transaction.atomic(): + project.github_last_commit = new_commit_sha + project.github_last_sync = now() + project.save() + return False + + commit = repo.get_git_commit(new_commit_sha) + tree = repo.get_git_tree(commit.tree.sha, recursive=True) + + paths_notags = {get_root_path(x.path) for x in tree.tree} + + try: + root, manifest = parse_manifest_from_tree( + [GitProjectItem(repo, x) for x in tree.tree], project) + except ValueError as e: + raise ValueError("In manifest file: %s" % str(e)) + + validate_resources_against_tree(paths_notags, manifest, project, root) + + changed_files = comparison.files + _apply_delta_changes(project, repo, root, manifest, changed_files, new_commit_sha) + + send_td_event('cloudpebble_github_pull', data={ + 'data': {'repo': project.github_repo} + }, user=user) + + return True + + +def _apply_delta_changes(project, repo, root, manifest, changed_files, new_commit_sha=None): + """Apply incremental file changes to the project database without a full wipe. + + Given a list of changed files from GitHub's Compare API, creates, updates, + or deletes only the affected SourceFile and ResourceFile/ResourceVariant + records. All changes (including the github_last_commit / github_last_sync + update) are wrapped in a single atomic transaction, so the project either + advances fully to ``new_commit_sha`` or not at all. + """ + manifest_kind = 'package.json' if 'pebble' in manifest else 'appinfo.json' + resource_root = ((root + '/' if root else '') + project.resources_path).rstrip('/') + '/' + + with transaction.atomic(): + project_options, media_map, dependencies = load_manifest_dict(manifest, manifest_kind) + + for k, v in project_options.items(): + setattr(project, k, v) + project.full_clean() + project.set_dependencies(dependencies) + + tag_map = {v: k for k, v in ResourceVariant.VARIANT_STRINGS.items() if v} + + existing_sources = {s.project_path: s for s in project.source_files.all()} + existing_resources = {} + for r in project.resources.all(): + dir_prefix = ResourceFile.DIR_MAP.get(r.kind, '') + '/' + root_file_name = dir_prefix + r.file_name if r.kind in ResourceFile.DIR_MAP else r.file_name + existing_resources[root_file_name] = r + + for change in changed_files: + filename = change.filename + status = change.status + project_path = filename[len(root) + 1:] if root and filename.startswith(root + '/') else filename + + if status in ('added', 'modified', 'renamed'): + if status == 'renamed' and change.previous_filename: + prev_project_path = change.previous_filename[len(root) + 1:] if root and change.previous_filename.startswith(root + '/') else change.previous_filename + _remove_file_by_path(project, prev_project_path, existing_sources, existing_resources) + + if project_path.startswith(project.resources_path + '/'): + _upsert_resource_variant(project, repo, change, project_path, existing_resources, tag_map, media_map) + else: + try: + base_filename, target = SourceFile.get_details_for_path(project.project_type, project_path) + _upsert_source_file(project, repo, change, base_filename, target, existing_sources, project_path) + except ValueError: + logger.debug("Skipping unrecognized file in delta: %s", filename) + continue + + elif status == 'removed': + _remove_file_by_path(project, project_path, existing_sources, existing_resources) + + _sync_resource_files_from_manifest(project, media_map, existing_resources) + + if new_commit_sha is not None: + project.github_last_commit = new_commit_sha + project.github_last_sync = now() + + project.save() + + +def _upsert_source_file(project, repo, change, base_filename, target, existing_sources, project_path=None): + """Create or update a SourceFile from a changed file in a GitHub comparison.""" + content = _fetch_file_content(repo, change) + if content is None: + logger.warning("Could not fetch content for %s, skipping", change.filename) + return + + if project_path is None: + project_path = change.filename + if project_path in existing_sources: + source = existing_sources[project_path] + else: + source = SourceFile.objects.create(project=project, file_name=base_filename, target=target) + existing_sources[project_path] = source + + if source.is_editable_text: + source.save_text(content.decode('utf-8') if isinstance(content, bytes) else content) + else: + source.save_string(content) + + +def _upsert_resource_variant(project, repo, change, project_path, existing_resources, tag_map, media_map=None): + """Create or update a ResourceVariant from a changed resource file in a GitHub comparison.""" + resource_root = project.resources_path + '/' + base_filename = project_path[len(resource_root):] + try: + tags, root_file_name = get_filename_variant(base_filename, tag_map) + except ValueError: + root_file_name = os.path.splitext(base_filename.split('~', 1)[0])[0] + os.path.splitext(base_filename)[1] + tags = [] + tags_string = ",".join(str(int(t)) for t in tags) + + if root_file_name in existing_resources: + resource_file = existing_resources[root_file_name] + else: + kind = _infer_resource_kind_from_path(root_file_name) + if media_map: + for resource in media_map: + try: + _, manifest_root = get_filename_variant(resource['file'], tag_map) + except ValueError: + manifest_root = os.path.splitext(resource['file'].split('~', 1)[0])[0] + os.path.splitext(resource['file'])[1] + if manifest_root == root_file_name: + kind = resource['type'] + break + file_name = _strip_resource_dir_prefix(root_file_name) + resource_file = ResourceFile.objects.create( + project=project, file_name=file_name, kind=kind) + existing_resources[root_file_name] = resource_file + + content = _fetch_file_content(repo, change) + if content is None: + logger.warning("Could not fetch content for resource %s, skipping", change.filename) + return + + variant = ResourceVariant.objects.filter( + resource_file=resource_file, tags=tags_string).first() + if variant is None: + variant = ResourceVariant.objects.create(resource_file=resource_file, tags=tags_string) + + variant.save_file(io.BytesIO(content)) + + +def _remove_file_by_path(project, filename, existing_sources, existing_resources): + """Remove a SourceFile or ResourceFile/Variant by its repo path.""" + resource_root = project.resources_path + '/' + tag_map = {v: k for k, v in ResourceVariant.VARIANT_STRINGS.items() if v} + + if filename.startswith(resource_root): + base_filename = filename[len(resource_root):] + try: + tags, root_file_name = get_filename_variant(base_filename, tag_map) + except ValueError: + return + + if root_file_name in existing_resources: + resource_file = existing_resources[root_file_name] + tags_string = ",".join(str(int(t)) for t in tags) + ResourceVariant.objects.filter( + resource_file=resource_file, tags=tags_string).delete() + if resource_file.variants.count() == 0: + del existing_resources[root_file_name] + resource_file.delete() + else: + try: + base_filename, target = SourceFile.get_details_for_path(project.project_type, filename) + except ValueError: + return + if filename in existing_sources: + existing_sources[filename].delete() + del existing_sources[filename] + else: + SourceFile.objects.filter( + project=project, file_name=base_filename, target=target).delete() + + +def _sync_resource_files_from_manifest(project, media_map, existing_resources): + """Reconcile ResourceFile and ResourceIdentifier records with the manifest. + + Creates new ResourceFile entries for resources in the manifest that don't + yet exist, creates ResourceIdentifier entries for each resource, and removes + resources no longer in the manifest. + """ + all_dir_prefixes = set(v + '/' for v in ResourceFile.DIR_MAP.values()) + desired_file_names = set() + + for resource in media_map: + if project.project_type in {'pebblejs', 'simplyjs'}: + if resource['name'] in PEBBLEJS_BUILTIN_RESOURCES: + continue + + file_name = resource['file'] + try: + tags, root_file_name = get_filename_variant(file_name, {v: k for k, v in ResourceVariant.VARIANT_STRINGS.items() if v}) + except ValueError: + root_file_name = os.path.splitext(file_name.split('~', 1)[0])[0] + os.path.splitext(file_name)[1] + + bare_name = _strip_resource_dir_prefix(root_file_name, all_dir_prefixes) + desired_file_names.add(root_file_name) + + if root_file_name not in existing_resources: + resource_file = ResourceFile.objects.create( + project=project, + file_name=bare_name, + kind=resource['type'], + is_menu_icon=resource.get('menuIcon', False), + ) + existing_resources[root_file_name] = resource_file + else: + resource_file = existing_resources[root_file_name] + if resource_file.kind != resource['type'] or resource_file.is_menu_icon != resource.get('menuIcon', False): + resource_file.kind = resource['type'] + resource_file.is_menu_icon = resource.get('menuIcon', False) + resource_file.save() + + resource_file = existing_resources[root_file_name] + target_platforms = json.dumps(resource['targetPlatforms']) if 'targetPlatforms' in resource else None + + ResourceIdentifier.objects.update_or_create( + resource_file=resource_file, + resource_id=resource['name'], + defaults={ + 'character_regex': resource.get('characterRegex', None), + 'tracking': resource.get('trackingAdjust', None), + 'compatibility': resource.get('compatibility', None), + 'memory_format': resource.get('memoryFormat', None), + 'storage_format': resource.get('storageFormat', None), + 'space_optimisation': resource.get('spaceOptimization', None), + 'target_platforms': target_platforms, + } + ) + + for file_name in list(existing_resources.keys()): + if file_name not in desired_file_names: + existing_resources[file_name].delete() + del existing_resources[file_name] + + +def _fetch_file_content(repo, change): + """Fetch the content of a file from GitHub, handling both text and binary files.""" + filename = change.filename + try: + contents = repo.get_contents(filename, ref=change.sha if hasattr(change, 'sha') and change.sha else None) + if isinstance(contents, list): + logger.warning("Expected file but got directory at %s, skipping", filename) + return None + if contents.encoding == 'base64': + return base64.b64decode(contents.content) + return contents.decoded_content + except GithubException as e: + logger.warning("Failed to fetch %s from GitHub: %s", filename, e) + return None + + +def _infer_resource_kind_from_path(filename): + """Infer the resource kind from the file extension.""" + ext = os.path.splitext(filename)[1].lower() + kind_map = { + '.png': 'png', + '.pbi': 'pbi', + '.ttf': 'font', + '.otf': 'font', + '.woff': 'font', + } + return kind_map.get(ext, 'raw') + + +def _strip_resource_dir_prefix(file_name, all_dir_prefixes=None): + """Strip resource directory prefix (e.g. 'images/', 'fonts/') from a file name. + + If all_dir_prefixes is not given, uses the default ResourceFile.DIR_MAP prefixes. + """ + if all_dir_prefixes is None: + all_dir_prefixes = set(v + '/' for v in ResourceFile.DIR_MAP.values()) + for prefix in all_dir_prefixes: + if file_name.startswith(prefix): + return file_name[len(prefix):] + return file_name + + @shared_task def do_github_push(project_id, commit_message): project = Project.objects.select_related('owner__github').get(pk=project_id) @@ -388,9 +731,20 @@ def do_github_push(project_id, commit_message): @shared_task -def do_github_pull(project_id): +def do_github_pull(project_id, force=False): project = Project.objects.select_related('owner__github').get(pk=project_id) - return github_pull(project.owner, project) + publish_event(project_id, 'pull_start') + try: + changed = github_pull(project.owner, project, force=force) + publish_event(project_id, 'pull_complete', github_last_commit=project.github_last_commit or '') + except Exception: + publish_event(project_id, 'pull_failed') + raise + + if changed and project.github_hook_build: + build = BuildResult.objects.create(project=project) + publish_event(project_id, 'build_start', build_id=build.id) + run_compile(build.id) @shared_task @@ -399,11 +753,18 @@ def hooked_commit(project_id, target_commit): did_something = False logger.debug("Comparing %s versus %s", project.github_last_commit, target_commit) if project.github_last_commit != target_commit: - github_pull(project.owner, project) + publish_event(project_id, 'pull_start') + try: + github_pull(project.owner, project, force=project.github_hook_force) + publish_event(project_id, 'pull_complete', github_last_commit=project.github_last_commit or '', github_last_sync=str(project.github_last_sync) if project.github_last_sync else '') + except Exception: + publish_event(project_id, 'pull_failed') + raise did_something = True if project.github_hook_build: build = BuildResult.objects.create(project=project) + publish_event(project_id, 'build_start', build_id=build.id) run_compile(build.id) did_something = True diff --git a/cloudpebble/ide/templates/ide/project.html b/cloudpebble/ide/templates/ide/project.html index 1516620..c41d1ba 100644 --- a/cloudpebble/ide/templates/ide/project.html +++ b/cloudpebble/ide/templates/ide/project.html @@ -357,14 +357,14 @@

{% trans 'Pull Latest Commit' %}

+
+ +
+ +
+
+
+ +
+
@@ -46,5 +57,6 @@
+
-
+ \ No newline at end of file diff --git a/cloudpebble/ide/tests/test_git.py b/cloudpebble/ide/tests/test_git.py index b62c5e1..96235d8 100644 --- a/cloudpebble/ide/tests/test_git.py +++ b/cloudpebble/ide/tests/test_git.py @@ -2,8 +2,42 @@ Tests in this file can be run with run_tests.py """ +import json + from django.test import TestCase +from unittest import mock import ide.git +from ide.tasks.git import ( + validate_resources_against_tree, + parse_manifest_from_tree, + get_root_path, + PEBBLEJS_BUILTIN_RESOURCES, + _infer_resource_kind_from_path, + _strip_resource_dir_prefix, + _fetch_file_content, + _remove_file_by_path, + github_pull, + _github_pull_delta, + _github_pull_full, + _apply_delta_changes, + _upsert_source_file, + _upsert_resource_variant, + _sync_resource_files_from_manifest, +) +from ide.utils.project import BaseProjectItem + + +class FakeItem(BaseProjectItem): + def __init__(self, item_path, content): + self._path = item_path + self._content = content + + def read(self): + return self._content + + @property + def path(self): + return self._path class UrlToReposTest(TestCase): @@ -28,3 +62,1326 @@ def test_bad_url_to_repo(self): Tests that a entirely different url returns None. """ self.assertEqual(None, ide.git.url_to_repo("http://www.cuteoverload.com")) + + +class GetRootPathTest(TestCase): + def test_strips_tilde_variant_suffix(self): + self.assertEqual(get_root_path('images/icon~color.png'), 'images/icon.png') + + def test_strips_multiple_tilde_variants(self): + self.assertEqual(get_root_path('images/icon~bw~rect.png'), 'images/icon.png') + + def test_no_variant(self): + self.assertEqual(get_root_path('images/icon.png'), 'images/icon.png') + + def test_no_extension(self): + self.assertEqual(get_root_path('data/binary~aplite'), 'data/binary') + + +class ValidateResourcesAgainstTreeTest(TestCase): + def _make_project(self, project_type='native', resources_path='resources'): + project = mock.MagicMock() + project.project_type = project_type + project.resources_path = resources_path + return project + + def test_all_resources_present_passes(self): + paths_notags = {'resources/images/icon.png', 'src/main.c'} + manifest = { + 'resources': {'media': [ + {'file': 'images/icon.png', 'name': 'ICON', 'type': 'png'} + ]} + } + project = self._make_project() + result = validate_resources_against_tree(paths_notags, manifest, project) + self.assertEqual(len(result), 1) + self.assertEqual(result[0]['name'], 'ICON') + self.assertEqual(result[0]['file'], 'images/icon.png') + + def test_multiple_resources_all_present(self): + paths_notags = {'resources/images/icon.png', 'resources/fonts/mono.ttf', 'src/main.c'} + manifest = { + 'resources': {'media': [ + {'file': 'images/icon.png', 'name': 'ICON', 'type': 'png'}, + {'file': 'fonts/mono.ttf', 'name': 'MONO', 'type': 'font'}, + ]} + } + project = self._make_project() + result = validate_resources_against_tree(paths_notags, manifest, project) + self.assertEqual(len(result), 2) + names = {r['name'] for r in result} + self.assertEqual(names, {'ICON', 'MONO'}) + + def test_missing_resource_raises(self): + paths_notags = {'src/main.c'} + manifest = { + 'resources': {'media': [ + {'file': 'images/icon.png', 'name': 'ICON', 'type': 'png'} + ]} + } + project = self._make_project() + with self.assertRaises(Exception) as ctx: + validate_resources_against_tree(paths_notags, manifest, project) + self.assertIn('images/icon.png', str(ctx.exception)) + + def test_missing_resource_with_variant_in_tree_still_fails(self): + paths_notags = {'resources/images/icon~color.png', 'src/main.c'} + manifest = { + 'resources': {'media': [ + {'file': 'images/icon.png', 'name': 'ICON', 'type': 'png'} + ]} + } + project = self._make_project() + with self.assertRaises(Exception) as ctx: + validate_resources_against_tree(paths_notags, manifest, project) + self.assertIn('images/icon.png', str(ctx.exception)) + + def test_pebblejs_skips_builtin_resources(self): + paths_notags = {'src/app.js'} + manifest = { + 'projectType': 'pebblejs', + 'resources': {'media': [ + {'file': 'images/mono.png', 'name': 'MONO_FONT_14', 'type': 'font'}, + {'file': 'images/icon.png', 'name': 'IMAGE_MENU_ICON', 'type': 'bitmap'}, + ]} + } + project = self._make_project(project_type='pebblejs') + result = validate_resources_against_tree(paths_notags, manifest, project) + self.assertEqual(len(result), 2) + names = {r['name'] for r in result} + self.assertEqual(names, {'MONO_FONT_14', 'IMAGE_MENU_ICON'}) + + def test_pebblejs_requires_non_builtin_resource(self): + paths_notags = {'src/app.js'} + manifest = { + 'projectType': 'pebblejs', + 'resources': {'media': [ + {'file': 'images/mono.png', 'name': 'MONO_FONT_14', 'type': 'font'}, + {'file': 'images/custom.png', 'name': 'CUSTOM_ICON', 'type': 'bitmap'}, + ]} + } + project = self._make_project(project_type='pebblejs') + with self.assertRaises(Exception) as ctx: + validate_resources_against_tree(paths_notags, manifest, project) + self.assertIn('custom.png', str(ctx.exception)) + self.assertNotIn('mono.png', str(ctx.exception)) + + def test_empty_resources_passes(self): + paths_notags = {'src/main.c'} + manifest = {'resources': {'media': []}} + project = self._make_project() + result = validate_resources_against_tree(paths_notags, manifest, project) + self.assertEqual(len(result), 0) + + def test_no_resources_key_passes(self): + paths_notags = {'src/main.c'} + manifest = {} + project = self._make_project() + result = validate_resources_against_tree(paths_notags, manifest, project) + self.assertEqual(len(result), 0) + + def test_package_project_uses_src_resources_prefix(self): + paths_notags = {'src/resources/data/config.json', 'src/main.c'} + manifest = { + 'resources': {'media': [ + {'file': 'data/config.json', 'name': 'CONFIG', 'type': 'raw'} + ]} + } + project = self._make_project(resources_path='src/resources') + result = validate_resources_against_tree(paths_notags, manifest, project) + self.assertEqual(len(result), 1) + self.assertEqual(result[0]['name'], 'CONFIG') + + def test_resource_with_variant_in_tree_matches_root(self): + paths_notags = {'resources/images/icon.png', 'src/main.c'} + manifest = { + 'resources': {'media': [ + {'file': 'images/icon.png', 'name': 'ICON', 'type': 'png'} + ]} + } + project = self._make_project() + result = validate_resources_against_tree(paths_notags, manifest, project) + self.assertEqual(len(result), 1) + self.assertEqual(result[0]['name'], 'ICON') + + def test_package_json_manifest_reads_resources_from_pebble_key(self): + paths_notags = {'src/resources/images/icon.png', 'src/main.c'} + manifest = { + 'pebble': { + 'projectType': 'native', + 'resources': {'media': [ + {'file': 'images/icon.png', 'name': 'ICON', 'type': 'png'} + ]} + } + } + project = self._make_project(resources_path='src/resources') + result = validate_resources_against_tree(paths_notags, manifest, project) + self.assertEqual(len(result), 1) + self.assertEqual(result[0]['name'], 'ICON') + + def test_package_json_pebble_skips_builtin_resources(self): + paths_notags = {'src/app.js'} + manifest = { + 'pebble': { + 'projectType': 'pebblejs', + 'resources': {'media': [ + {'file': 'images/mono.png', 'name': 'MONO_FONT_14', 'type': 'font'}, + {'file': 'images/custom.png', 'name': 'CUSTOM_ICON', 'type': 'bitmap'}, + ]} + } + } + project = self._make_project(project_type='pebblejs', resources_path='src/resources') + with self.assertRaises(Exception) as ctx: + validate_resources_against_tree(paths_notags, manifest, project) + self.assertIn('custom.png', str(ctx.exception)) + self.assertNotIn('mono.png', str(ctx.exception)) + + def test_root_prefix_prepended_to_paths(self): + paths_notags = {'myproject/resources/images/icon.png', 'myproject/src/main.c'} + manifest = { + 'resources': {'media': [ + {'file': 'images/icon.png', 'name': 'ICON', 'type': 'png'} + ]} + } + project = self._make_project() + result = validate_resources_against_tree(paths_notags, manifest, project, root='myproject') + self.assertEqual(len(result), 1) + self.assertEqual(result[0]['name'], 'ICON') + + def test_root_prefix_with_package_json_manifest(self): + paths_notags = {'sdk-demo/src/resources/data/config.json', 'sdk-demo/src/main.c'} + manifest = { + 'pebble': { + 'projectType': 'native', + 'resources': {'media': [ + {'file': 'data/config.json', 'name': 'CONFIG', 'type': 'raw'} + ]} + } + } + project = self._make_project(resources_path='src/resources') + result = validate_resources_against_tree(paths_notags, manifest, project, root='sdk-demo') + self.assertEqual(len(result), 1) + self.assertEqual(result[0]['name'], 'CONFIG') + + +class PebblejsBuiltinsTest(TestCase): + def test_builtin_resource_names(self): + self.assertEqual( + PEBBLEJS_BUILTIN_RESOURCES, + {'MONO_FONT_14', 'IMAGE_MENU_ICON', 'IMAGE_LOGO_SPLASH', 'IMAGE_TILE_SPLASH'}, + ) + + def test_builtin_resources_are_frozenset(self): + self.assertIsInstance(PEBBLEJS_BUILTIN_RESOURCES, frozenset) + + +class ParseManifestFromTreeTest(TestCase): + def _make_appinfo(self, **overrides): + appinfo = { + "uuid": "123e4567-e89b-42d3-a456-426655440000", + "longName": "test", + "shortName": "test", + "companyName": "test", + "versionLabel": "1.0", + "projectType": "native", + "sdkVersion": "3", + "enableMultiJS": True, + "watchapp": {"watchface": False}, + "appKeys": {}, + "resources": {"media": []}, + } + appinfo.update(overrides) + return json.dumps(appinfo) + + def _make_package(self, pebble_options=None, **overrides): + package = { + "name": "test", + "version": "1.0.0", + "author": "test", + "dependencies": {}, + "keywords": [], + "pebble": { + "displayName": "test", + "messageKeys": [], + "enableMultiJS": True, + "projectType": "native", + "resources": {"media": []}, + "sdkVersion": "3", + "uuid": "123e4567-e89b-42d3-a456-426655440000", + "watchapp": {"watchface": False}, + }, + } + if pebble_options: + package["pebble"].update(pebble_options) + package.update(overrides) + return json.dumps(package) + + def test_extracts_appinfo_manifest_from_root(self): + items = [ + FakeItem('src/main.c', 'int main(void) { return 0; }'), + FakeItem('appinfo.json', self._make_appinfo()), + ] + root, manifest = parse_manifest_from_tree(items, None) + self.assertEqual(root, '') + self.assertEqual(manifest['projectType'], 'native') + self.assertEqual(manifest['shortName'], 'test') + self.assertIn('resources', manifest) + + def test_extracts_appinfo_manifest_from_subdirectory(self): + items = [ + FakeItem('myproject/src/main.c', 'int main(void) { return 0; }'), + FakeItem('myproject/appinfo.json', self._make_appinfo()), + ] + root, manifest = parse_manifest_from_tree(items, None) + self.assertEqual(root, 'myproject/') + self.assertEqual(manifest['projectType'], 'native') + + def test_extracts_package_json_manifest(self): + items = [ + FakeItem('src/main.c', 'int main(void) { return 0; }'), + FakeItem('package.json', self._make_package()), + ] + root, manifest = parse_manifest_from_tree(items, None) + self.assertEqual(root, '') + self.assertIn('pebble', manifest) + self.assertEqual(manifest['pebble']['projectType'], 'native') + + def test_raises_on_invalid_json(self): + items = [ + FakeItem('src/main.c', 'int main(void) { return 0; }'), + FakeItem('appinfo.json', 'not valid json {{{'), + ] + from ide.utils.project import InvalidProjectArchiveException + with self.assertRaises(InvalidProjectArchiveException): + parse_manifest_from_tree(items, None) + + def test_raises_when_no_manifest_found(self): + items = [ + FakeItem('src/main.c', 'int main(void) { return 0; }'), + ] + from ide.utils.project import InvalidProjectArchiveException + with self.assertRaises(InvalidProjectArchiveException): + parse_manifest_from_tree(items, None) + + +class InferResourceKindTest(TestCase): + def test_png_maps_to_png(self): + self.assertEqual(_infer_resource_kind_from_path('icon.png'), 'png') + + def test_pbi_maps_to_pbi(self): + self.assertEqual(_infer_resource_kind_from_path('image.pbi'), 'pbi') + + def test_ttf_maps_to_font(self): + self.assertEqual(_infer_resource_kind_from_path('font.ttf'), 'font') + + def test_otf_maps_to_font(self): + self.assertEqual(_infer_resource_kind_from_path('font.otf'), 'font') + + def test_unknown_maps_to_raw(self): + self.assertEqual(_infer_resource_kind_from_path('data.bin'), 'raw') + + def test_jpg_maps_to_raw(self): + self.assertEqual(_infer_resource_kind_from_path('photo.jpg'), 'raw') + + +class StripResourceDirPrefixTest(TestCase): + def test_strips_images_prefix(self): + self.assertEqual(_strip_resource_dir_prefix('images/icon.png'), 'icon.png') + + def test_strips_fonts_prefix(self): + self.assertEqual(_strip_resource_dir_prefix('fonts/mono.ttf'), 'mono.ttf') + + def test_strips_data_prefix(self): + self.assertEqual(_strip_resource_dir_prefix('data/config.json'), 'config.json') + + def test_no_prefix_returns_unchanged(self): + self.assertEqual(_strip_resource_dir_prefix('icon.png'), 'icon.png') + + def test_custom_prefixes(self): + prefixes = {'images/', 'fonts/', 'data/'} + self.assertEqual(_strip_resource_dir_prefix('images/icon.png', prefixes), 'icon.png') + + +class FetchFileContentTest(TestCase): + def _make_change(self, filename, sha=None): + change = mock.MagicMock() + change.filename = filename + change.sha = sha + return change + + def test_returns_decoded_content_for_text_file(self): + repo = mock.MagicMock() + contents = mock.MagicMock() + contents.encoding = None + contents.decoded_content = b'hello world' + repo.get_contents.return_value = contents + + change = self._make_change('src/main.c', sha='abc123') + result = _fetch_file_content(repo, change) + self.assertEqual(result, b'hello world') + repo.get_contents.assert_called_once_with('src/main.c', ref='abc123') + + def test_returns_base64_decoded_content(self): + import base64 + repo = mock.MagicMock() + contents = mock.MagicMock() + contents.encoding = 'base64' + contents.content = base64.b64encode(b'binary data').decode('ascii') + repo.get_contents.return_value = contents + + change = self._make_change('resources/images/icon.png') + result = _fetch_file_content(repo, change) + self.assertEqual(result, b'binary data') + + def test_returns_none_for_directory(self): + repo = mock.MagicMock() + repo.get_contents.return_value = [mock.MagicMock(), mock.MagicMock()] + + change = self._make_change('src/') + result = _fetch_file_content(repo, change) + self.assertIsNone(result) + + def test_returns_none_on_github_exception(self): + from github import GithubException + repo = mock.MagicMock() + repo.get_contents.side_effect = GithubException(404, 'Not Found', {}) + + change = self._make_change('src/missing.c', sha='abc') + result = _fetch_file_content(repo, change) + self.assertIsNone(result) + + def test_uses_sha_ref_when_available(self): + repo = mock.MagicMock() + contents = mock.MagicMock() + contents.encoding = None + contents.decoded_content = b'data' + repo.get_contents.return_value = contents + + change = self._make_change('src/main.c', sha='deadbeef') + _fetch_file_content(repo, change) + repo.get_contents.assert_called_once_with('src/main.c', ref='deadbeef') + + def test_uses_no_ref_when_sha_is_none(self): + repo = mock.MagicMock() + contents = mock.MagicMock() + contents.encoding = None + contents.decoded_content = b'data' + repo.get_contents.return_value = contents + + change = self._make_change('src/main.c', sha=None) + _fetch_file_content(repo, change) + repo.get_contents.assert_called_once_with('src/main.c', ref=None) + + +class RemoveFileByPathTest(TestCase): + def _make_project(self, project_type='native', resources_path='resources'): + project = mock.MagicMock() + project.project_type = project_type + project.resources_path = resources_path + return project + + def test_removes_existing_source_file(self): + project = self._make_project() + source = mock.MagicMock() + existing_sources = {'src/main.c': source} + existing_resources = {} + + _remove_file_by_path(project, 'src/main.c', existing_sources, existing_resources) + source.delete.assert_called_once() + self.assertNotIn('src/main.c', existing_sources) + + def test_removes_missing_source_file_from_db(self): + project = self._make_project() + existing_sources = {} + existing_resources = {} + + with mock.patch('ide.tasks.git.SourceFile.get_details_for_path', return_value=('main.c', 'app')): + with mock.patch('ide.tasks.git.SourceFile.objects') as mock_objects: + mock_filter = mock.MagicMock() + mock_objects.filter.return_value = mock_filter + _remove_file_by_path(project, 'src/main.c', existing_sources, existing_resources) + mock_objects.filter.assert_called_once() + mock_filter.delete.assert_called_once() + + def test_removes_resource_variant_and_orphaned_file(self): + project = self._make_project() + resource_file = mock.MagicMock() + resource_file.variants.count.return_value = 0 + existing_sources = {} + existing_resources = {'images/icon.png': resource_file} + + with mock.patch('ide.tasks.git.ResourceVariant.objects') as mock_objects: + mock_filter = mock.MagicMock() + mock_objects.filter.return_value = mock_filter + _remove_file_by_path(project, 'resources/images/icon.png', existing_sources, existing_resources) + mock_objects.filter.assert_called_once() + mock_filter.delete.assert_called_once() + resource_file.delete.assert_called_once() + self.assertNotIn('images/icon.png', existing_resources) + + def test_removes_resource_variant_but_keeps_file_with_remaining_variants(self): + project = self._make_project() + resource_file = mock.MagicMock() + resource_file.variants.count.return_value = 1 + existing_sources = {} + existing_resources = {'images/icon.png': resource_file} + + with mock.patch('ide.tasks.git.ResourceVariant.objects') as mock_objects: + mock_filter = mock.MagicMock() + mock_objects.filter.return_value = mock_filter + _remove_file_by_path(project, 'resources/images/icon~color.png', existing_sources, existing_resources) + mock_objects.filter.assert_called_once() + mock_filter.delete.assert_called_once() + resource_file.delete.assert_not_called() + self.assertIn('images/icon.png', existing_resources) + + def test_skips_unrecognized_resource_tags(self): + project = self._make_project() + existing_sources = {} + existing_resources = {} + + _remove_file_by_path(project, 'resources/images/icon~unknowntag.png', existing_sources, existing_resources) + + def test_skips_unrecognized_source_paths(self): + project = self._make_project() + existing_sources = {} + existing_resources = {} + + with mock.patch('ide.tasks.git.SourceFile') as MockSourceFile: + MockSourceFile.get_details_for_path.side_effect = ValueError('bad path') + _remove_file_by_path(project, 'unknown/path.dat', existing_sources, existing_resources) + MockSourceFile.objects.filter.assert_not_called() + + +def _mock_github(): + mock_repo = mock.MagicMock() + mock_repo.default_branch = 'main' + mock_branch = mock.MagicMock() + mock_branch.commit.sha = 'newsha' + mock_repo.get_branch.return_value = mock_branch + return mock_repo, mock_branch + + +class GithubPullRoutingTest(TestCase): + def setUp(self): + self.user = mock.MagicMock() + self.project = mock.MagicMock() + self.project.github_repo = 'owner/repo' + self.project.github_branch = 'main' + self.project.github_last_commit = 'oldsha' + self.project.github_hook_force = False + + @mock.patch('ide.git.git_verify_tokens', return_value=True) + @mock.patch('ide.tasks.git._github_pull_full') + @mock.patch('ide.tasks.git.get_github') + def test_force_pull_uses_full(self, mock_get_github, mock_full, mock_verify): + mock_repo, mock_branch = _mock_github() + mock_get_github.return_value.get_repo.return_value = mock_repo + mock_full.return_value = True + + result = github_pull(self.user, self.project, force=True) + mock_full.assert_called_once() + self.assertTrue(result) + + @mock.patch('ide.git.git_verify_tokens', return_value=True) + @mock.patch('ide.tasks.git._github_pull_full') + @mock.patch('ide.tasks.git.get_github') + def test_no_previous_commit_uses_full(self, mock_get_github, mock_full, mock_verify): + self.project.github_last_commit = None + mock_repo, mock_branch = _mock_github() + mock_get_github.return_value.get_repo.return_value = mock_repo + mock_full.return_value = True + + result = github_pull(self.user, self.project, force=False) + mock_full.assert_called_once() + + @mock.patch('ide.git.git_verify_tokens', return_value=True) + @mock.patch('ide.tasks.git._github_pull_delta') + @mock.patch('ide.tasks.git._github_pull_full') + @mock.patch('ide.tasks.git.get_github') + def test_delta_sync_when_not_forced(self, mock_get_github, mock_full, mock_delta, mock_verify): + mock_repo, mock_branch = _mock_github() + mock_get_github.return_value.get_repo.return_value = mock_repo + mock_delta.return_value = True + + result = github_pull(self.user, self.project, force=False) + mock_delta.assert_called_once_with(self.user, self.project, mock_repo, 'newsha') + mock_full.assert_not_called() + self.assertTrue(result) + + @mock.patch('ide.git.git_verify_tokens', return_value=True) + @mock.patch('ide.tasks.git._github_pull_delta') + @mock.patch('ide.tasks.git._github_pull_full') + @mock.patch('ide.tasks.git.get_github') + def test_falls_back_to_full_on_delta_failure(self, mock_get_github, mock_full, mock_delta, mock_verify): + mock_repo, mock_branch = _mock_github() + mock_get_github.return_value.get_repo.return_value = mock_repo + mock_delta.side_effect = Exception('compare API failed') + mock_full.return_value = True + + result = github_pull(self.user, self.project, force=False) + mock_full.assert_called_once_with(self.user, self.project, mock_repo, mock_branch) + self.assertTrue(result) + + @mock.patch('ide.git.git_verify_tokens', return_value=True) + @mock.patch('ide.tasks.git.get_github') + def test_returns_false_when_no_new_commits(self, mock_get_github, mock_verify): + mock_repo = mock.MagicMock() + mock_repo.default_branch = 'main' + mock_branch = mock.MagicMock() + mock_branch.commit.sha = 'oldsha' + mock_repo.get_branch.return_value = mock_branch + mock_get_github.return_value.get_repo.return_value = mock_repo + + result = github_pull(self.user, self.project, force=False) + self.assertFalse(result) + + @mock.patch('ide.git.git_verify_tokens', return_value=True) + @mock.patch('ide.tasks.git.get_github') + def test_raises_when_no_repo_defined(self, mock_get_github, mock_verify): + self.project.github_repo = None + mock_repo, mock_branch = _mock_github() + mock_get_github.return_value.get_repo.return_value = mock_repo + + with self.assertRaises(Exception) as ctx: + github_pull(self.user, self.project, force=False) + self.assertIn('No GitHub repo defined', str(ctx.exception)) + + @mock.patch('ide.git.git_verify_tokens', return_value=True) + @mock.patch('ide.tasks.git.get_github') + def test_raises_when_branch_not_found(self, mock_get_github, mock_verify): + from github import GithubException + mock_repo = mock.MagicMock() + mock_repo.default_branch = 'main' + mock_repo.get_branch.side_effect = GithubException(404, 'Not Found', {}) + mock_get_github.return_value.get_repo.return_value = mock_repo + + with self.assertRaises(Exception) as ctx: + github_pull(self.user, self.project, force=False) + self.assertIn('Unable to get the branch', str(ctx.exception)) + + +class GithubPullDeltaTest(TestCase): + def setUp(self): + self.user = mock.MagicMock() + self.project = mock.MagicMock() + self.project.github_last_commit = 'oldsha' + self.project.resources_path = 'resources' + self.project.project_type = 'native' + self.repo = mock.MagicMock() + + @mock.patch('ide.tasks.git._apply_delta_changes') + @mock.patch('ide.tasks.git.validate_resources_against_tree') + @mock.patch('ide.tasks.git.parse_manifest_from_tree') + @mock.patch('ide.tasks.git.get_root_path') + @mock.patch('ide.tasks.git.now') + def test_delta_pull_applies_changes(self, mock_now, mock_get_root, mock_parse, mock_validate, mock_apply): + mock_now.return_value = '2025-01-01T00:00:00Z' + comparison = mock.MagicMock() + comparison.ahead_by = 3 + comparison.files = [mock.MagicMock(filename='src/main.c', status='modified')] + self.repo.compare.return_value = comparison + + mock_commit = mock.MagicMock() + mock_commit.tree.sha = 'treesha' + self.repo.get_git_commit.return_value = mock_commit + mock_tree = mock.MagicMock() + mock_tree.tree = [] + self.repo.get_git_tree.return_value = mock_tree + + mock_parse.return_value = ('', {'projectType': 'native'}) + mock_get_root.return_value = 'src/main.c' + + # _apply_delta_changes is responsible for stamping the new SHA inside + # its atomic block; simulate that here. + def fake_apply(project, repo, root, manifest, changed_files, new_commit_sha=None): + project.github_last_commit = new_commit_sha + project.github_last_sync = mock_now.return_value + project.save() + mock_apply.side_effect = fake_apply + + result = _github_pull_delta(self.user, self.project, self.repo, 'newsha') + self.assertTrue(result) + mock_apply.assert_called_once() + self.assertEqual(self.project.github_last_commit, 'newsha') + self.project.save.assert_called() + + @mock.patch('ide.tasks.git.now') + def test_delta_pull_returns_false_when_ahead_by_zero(self, mock_now): + mock_now.return_value = '2025-01-01T00:00:00Z' + comparison = mock.MagicMock() + comparison.ahead_by = 0 + self.repo.compare.return_value = comparison + + result = _github_pull_delta(self.user, self.project, self.repo, 'newsha') + self.assertFalse(result) + self.assertEqual(self.project.github_last_commit, 'newsha') + + +class ApplyDeltaChangesTest(TestCase): + def _make_change(self, filename, status, sha=None, previous_filename=None): + change = mock.MagicMock() + change.filename = filename + change.status = status + change.sha = sha or 'abc123' + if previous_filename: + change.previous_filename = previous_filename + else: + change.previous_filename = None + return change + + @mock.patch('ide.tasks.git._sync_resource_files_from_manifest') + @mock.patch('ide.tasks.git.load_manifest_dict') + @mock.patch('ide.tasks.git._upsert_source_file') + def test_applies_added_source_file(self, mock_upsert, mock_load, mock_sync): + mock_load.return_value = ({}, {}, {}) + project = mock.MagicMock() + project.resources_path = 'resources' + project.project_type = 'native' + repo = mock.MagicMock() + + change = self._make_change('src/main.c', 'added') + manifest = {'projectType': 'native', 'resources': {'media': []}} + + with mock.patch('ide.tasks.git.transaction'): + _apply_delta_changes(project, repo, '', manifest, [change]) + + mock_upsert.assert_called_once() + self.assertEqual(mock_upsert.call_args[0][2].filename, 'src/main.c') + + @mock.patch('ide.tasks.git._sync_resource_files_from_manifest') + @mock.patch('ide.tasks.git.load_manifest_dict') + @mock.patch('ide.tasks.git._remove_file_by_path') + def test_applies_removed_source_file(self, mock_remove, mock_load, mock_sync): + mock_load.return_value = ({}, {}, {}) + project = mock.MagicMock() + project.resources_path = 'resources' + project.project_type = 'native' + repo = mock.MagicMock() + + change = self._make_change('src/main.c', 'removed') + manifest = {'projectType': 'native', 'resources': {'media': []}} + existing_sources = {} + existing_resources = {} + + with mock.patch('ide.tasks.git.transaction'): + with mock.patch('ide.tasks.git.SourceFile') as MockSF: + mock_qs = mock.MagicMock() + project.source_files.all.return_value = [] + project.resources.all.return_value = [] + _apply_delta_changes(project, repo, '', manifest, [change]) + + mock_remove.assert_called() + + @mock.patch('ide.tasks.git._sync_resource_files_from_manifest') + @mock.patch('ide.tasks.git.load_manifest_dict') + @mock.patch('ide.tasks.git._remove_file_by_path') + @mock.patch('ide.tasks.git._upsert_source_file') + def test_handles_renamed_file(self, mock_upsert, mock_remove, mock_load, mock_sync): + mock_load.return_value = ({}, {}, {}) + project = mock.MagicMock() + project.resources_path = 'resources' + project.project_type = 'native' + repo = mock.MagicMock() + + change = self._make_change('src/new_main.c', 'renamed', previous_filename='src/old_main.c') + manifest = {'projectType': 'native', 'resources': {'media': []}} + + with mock.patch('ide.tasks.git.transaction'): + with mock.patch('ide.tasks.git.SourceFile') as MockSF: + project.source_files.all.return_value = [] + project.resources.all.return_value = [] + _apply_delta_changes(project, repo, '', manifest, [change]) + + mock_remove.assert_called_once() + self.assertEqual(mock_remove.call_args[0][1], 'src/old_main.c') + + @mock.patch('ide.tasks.git._sync_resource_files_from_manifest') + @mock.patch('ide.tasks.git.load_manifest_dict') + @mock.patch('ide.tasks.git.SourceFile') + def test_skips_unrecognized_source_file_path(self, MockSF, mock_load, mock_sync): + mock_load.return_value = ({}, {}, {}) + project = mock.MagicMock() + project.resources_path = 'resources' + project.project_type = 'native' + repo = mock.MagicMock() + + change = self._make_change('unknown/weird.dat', 'added') + manifest = {'projectType': 'native', 'resources': {'media': []}} + + MockSF.get_details_for_path.side_effect = ValueError('bad path') + + with mock.patch('ide.tasks.git.transaction'): + project.source_files.all.return_value = [] + project.resources.all.return_value = [] + _apply_delta_changes(project, repo, '', manifest, [change]) + + MockSF.objects.create.assert_not_called() + + @mock.patch('ide.tasks.git._upsert_resource_variant') + @mock.patch('ide.tasks.git._sync_resource_files_from_manifest') + @mock.patch('ide.tasks.git.load_manifest_dict') + def test_routes_resource_file_to_upsert_resource_variant(self, mock_load, mock_sync, mock_upsert_resource): + mock_load.return_value = ({}, {}, {}) + project = mock.MagicMock() + project.resources_path = 'resources' + project.project_type = 'native' + repo = mock.MagicMock() + + change = self._make_change('resources/images/icon.png', 'added') + manifest = {'projectType': 'native', 'resources': {'media': []}} + tag_map = {} + + with mock.patch('ide.tasks.git.transaction'): + with mock.patch('ide.tasks.git.ResourceVariant') as MockRV: + mock_rv_map = {v: k for k, v in MockRV.VARIANT_STRINGS.items() if v} + with mock.patch('ide.tasks.git.SourceFile'): + project.source_files.all.return_value = [] + project.resources.all.return_value = [] + _apply_delta_changes(project, repo, '', manifest, [change]) + + mock_upsert_resource.assert_called_once() + + @mock.patch('ide.tasks.git._sync_resource_files_from_manifest') + @mock.patch('ide.tasks.git.load_manifest_dict') + @mock.patch('ide.tasks.git._upsert_resource_variant') + @mock.patch('ide.tasks.git.SourceFile') + def test_existing_resources_keyed_by_manifest_path(self, MockSF, mock_upsert_resource, mock_load, mock_sync): + mock_load.return_value = ({}, {}, {}) + project = mock.MagicMock() + project.resources_path = 'resources' + project.project_type = 'native' + repo = mock.MagicMock() + + existing_resource = mock.MagicMock() + existing_resource.file_name = 'icon.png' + existing_resource.kind = 'png' + project.resources.all.return_value = [existing_resource] + project.source_files.all.return_value = [] + + change = self._make_change('resources/images/icon.png', 'modified') + manifest = {'projectType': 'native', 'resources': {'media': []}} + + with mock.patch('ide.tasks.git.transaction'): + with mock.patch('ide.tasks.git.ResourceVariant') as MockRV: + mock_rv_map = {v: k for k, v in MockRV.VARIANT_STRINGS.items() if v} + _apply_delta_changes(project, repo, '', manifest, [change]) + + mock_upsert_resource.assert_called_once() + self.assertIs(mock_upsert_resource.call_args[0][4]['images/icon.png'], existing_resource) + + @mock.patch('ide.tasks.git._sync_resource_files_from_manifest') + @mock.patch('ide.tasks.git.load_manifest_dict') + def test_updates_project_options_from_manifest(self, mock_load, mock_sync): + mock_load.return_value = ({ + 'app_long_name': 'My App', + 'app_short_name': 'MyApp', + }, {}, {}) + project = mock.MagicMock() + project.resources_path = 'resources' + project.project_type = 'native' + repo = mock.MagicMock() + + manifest = {'projectType': 'native', 'resources': {'media': []}} + + with mock.patch('ide.tasks.git.transaction'): + project.source_files.all.return_value = [] + project.resources.all.return_value = [] + _apply_delta_changes(project, repo, '', manifest, []) + + project.full_clean.assert_called_once() + project.set_dependencies.assert_called_once_with({}) + + +class UpsertSourceFileTest(TestCase): + def test_creates_new_source_file_when_not_in_existing(self): + project = mock.MagicMock() + repo = mock.MagicMock() + existing_sources = {} + + change = mock.MagicMock() + change.filename = 'src/main.c' + change.sha = 'abc123' + + mock_source = mock.MagicMock() + mock_source.is_editable_text = True + + contents = mock.MagicMock() + contents.encoding = None + contents.decoded_content = b'// hello' + repo.get_contents.return_value = contents + + with mock.patch('ide.tasks.git.SourceFile') as MockSF: + MockSF.objects.create.return_value = mock_source + MockSF.get_details_for_path.return_value = ('main.c', 'app') + _upsert_source_file(project, repo, change, 'main.c', 'app', existing_sources) + + MockSF.objects.create.assert_called_once_with(project=project, file_name='main.c', target='app') + mock_source.save_text.assert_called_once_with('// hello') + + def test_updates_existing_source_file(self): + project = mock.MagicMock() + repo = mock.MagicMock() + existing_source = mock.MagicMock() + existing_source.is_editable_text = True + existing_sources = {'src/main.c': existing_source} + + change = mock.MagicMock() + change.filename = 'src/main.c' + change.sha = 'def456' + + contents = mock.MagicMock() + contents.encoding = None + contents.decoded_content = b'// updated' + repo.get_contents.return_value = contents + + _upsert_source_file(project, repo, change, 'main.c', 'app', existing_sources) + + existing_source.save_text.assert_called_once_with('// updated') + + @mock.patch('ide.tasks.git._fetch_file_content') + def test_skips_when_content_is_none(self, mock_fetch): + project = mock.MagicMock() + repo = mock.MagicMock() + existing_sources = {} + + change = mock.MagicMock() + change.filename = 'src/main.c' + + mock_fetch.return_value = None + + with mock.patch('ide.tasks.git.SourceFile') as MockSF: + _upsert_source_file(project, repo, change, 'main.c', 'app', existing_sources) + MockSF.objects.create.assert_not_called() + + +class UpsertResourceVariantTest(TestCase): + def test_creates_new_resource_variant(self): + project = mock.MagicMock() + project.resources_path = 'resources' + repo = mock.MagicMock() + + change = mock.MagicMock() + change.filename = 'resources/images/icon.png' + change.sha = 'abc123' + + existing_resources = {} + tag_map = {} + + contents = mock.MagicMock() + contents.encoding = None + contents.decoded_content = b'\x89PNG' + repo.get_contents.return_value = contents + + mock_resource = mock.MagicMock() + mock_variant = mock.MagicMock() + + with mock.patch('ide.tasks.git.ResourceFile') as MockRF: + with mock.patch('ide.tasks.git.ResourceVariant') as MockRV: + with mock.patch('ide.tasks.git.ResourceVariant.VARIANT_STRINGS', {}): + MockRF.objects.create.return_value = mock_resource + MockRV.objects.filter.return_value.first.return_value = None + MockRV.objects.create.return_value = mock_variant + + _upsert_resource_variant(project, repo, change, 'resources/images/icon.png', existing_resources, tag_map) + + MockRF.objects.create.assert_called_once() + mock_variant.save_file.assert_called_once() + + def test_adds_variant_to_existing_resource(self): + project = mock.MagicMock() + project.resources_path = 'resources' + repo = mock.MagicMock() + + change = mock.MagicMock() + change.filename = 'resources/images/icon.png' + change.sha = 'abc123' + + existing_resource = mock.MagicMock() + existing_resources = {'images/icon.png': existing_resource} + tag_map = {} + + contents = mock.MagicMock() + contents.encoding = None + contents.decoded_content = b'\x89PNG' + repo.get_contents.return_value = contents + + mock_variant = mock.MagicMock() + + with mock.patch('ide.tasks.git.ResourceFile') as MockRF: + with mock.patch('ide.tasks.git.ResourceVariant') as MockRV: + with mock.patch('ide.tasks.git.ResourceVariant.VARIANT_STRINGS', {}): + MockRV.objects.filter.return_value.first.return_value = None + MockRV.objects.create.return_value = mock_variant + + _upsert_resource_variant(project, repo, change, 'resources/images/icon.png', existing_resources, tag_map) + + MockRF.objects.create.assert_not_called() + mock_variant.save_file.assert_called_once() + + +class ApplyDeltaChangesWithRootTest(TestCase): + def _make_change(self, filename, status, sha=None, previous_filename=None): + change = mock.MagicMock() + change.filename = filename + change.status = status + change.sha = sha or 'abc123' + if previous_filename: + change.previous_filename = previous_filename + else: + change.previous_filename = None + return change + + @mock.patch('ide.tasks.git._sync_resource_files_from_manifest') + @mock.patch('ide.tasks.git.load_manifest_dict') + @mock.patch('ide.tasks.git._upsert_source_file') + def test_strips_root_from_source_file_path(self, mock_upsert, mock_load, mock_sync): + mock_load.return_value = ({}, {}, {}) + project = mock.MagicMock() + project.resources_path = 'resources' + project.project_type = 'native' + repo = mock.MagicMock() + + change = self._make_change('myproject/src/main.c', 'added') + manifest = {'projectType': 'native', 'resources': {'media': []}} + + with mock.patch('ide.tasks.git.transaction'): + with mock.patch('ide.tasks.git.SourceFile') as MockSF: + MockSF.get_details_for_path.return_value = ('main.c', 'app') + project.source_files.all.return_value = [] + project.resources.all.return_value = [] + _apply_delta_changes(project, repo, 'myproject', manifest, [change]) + + mock_upsert.assert_called_once() + self.assertEqual(mock_upsert.call_args[0][6], 'src/main.c') + + @mock.patch('ide.tasks.git._sync_resource_files_from_manifest') + @mock.patch('ide.tasks.git.load_manifest_dict') + @mock.patch('ide.tasks.git._upsert_resource_variant') + def test_strips_root_from_resource_file_path(self, mock_upsert_resource, mock_load, mock_sync): + mock_load.return_value = ({}, {}, {}) + project = mock.MagicMock() + project.resources_path = 'resources' + project.project_type = 'native' + repo = mock.MagicMock() + + change = self._make_change('myproject/resources/images/icon.png', 'added') + manifest = {'projectType': 'native', 'resources': {'media': []}} + + with mock.patch('ide.tasks.git.transaction'): + with mock.patch('ide.tasks.git.ResourceVariant') as MockRV: + mock_rv_map = {v: k for k, v in MockRV.VARIANT_STRINGS.items() if v} + with mock.patch('ide.tasks.git.SourceFile'): + project.source_files.all.return_value = [] + project.resources.all.return_value = [] + _apply_delta_changes(project, repo, 'myproject', manifest, [change]) + + mock_upsert_resource.assert_called_once() + self.assertEqual(mock_upsert_resource.call_args[0][3], 'resources/images/icon.png') + + @mock.patch('ide.tasks.git._sync_resource_files_from_manifest') + @mock.patch('ide.tasks.git.load_manifest_dict') + @mock.patch('ide.tasks.git._remove_file_by_path') + @mock.patch('ide.tasks.git._upsert_source_file') + def test_strips_root_from_renamed_previous_filename(self, mock_upsert, mock_remove, mock_load, mock_sync): + mock_load.return_value = ({}, {}, {}) + project = mock.MagicMock() + project.resources_path = 'resources' + project.project_type = 'native' + repo = mock.MagicMock() + + change = self._make_change('myproject/src/new_main.c', 'renamed', previous_filename='myproject/src/old_main.c') + manifest = {'projectType': 'native', 'resources': {'media': []}} + + with mock.patch('ide.tasks.git.transaction'): + with mock.patch('ide.tasks.git.SourceFile') as MockSF: + MockSF.get_details_for_path.return_value = ('new_main.c', 'app') + project.source_files.all.return_value = [] + project.resources.all.return_value = [] + _apply_delta_changes(project, repo, 'myproject', manifest, [change]) + + mock_remove.assert_called_once() + + +class UpsertResourceVariantKindOverrideTest(TestCase): + def test_uses_manifest_type_when_media_map_provided(self): + project = mock.MagicMock() + project.resources_path = 'resources' + repo = mock.MagicMock() + + change = mock.MagicMock() + change.filename = 'resources/images/icon.png' + change.sha = 'abc123' + + existing_resources = {} + tag_map = {} + media_map = [{'file': 'images/icon.png', 'name': 'ICON', 'type': 'bitmap'}] + + contents = mock.MagicMock() + contents.encoding = None + contents.decoded_content = b'\x89PNG' + repo.get_contents.return_value = contents + + mock_resource = mock.MagicMock() + mock_variant = mock.MagicMock() + + with mock.patch('ide.tasks.git.ResourceFile') as MockRF: + with mock.patch('ide.tasks.git.ResourceVariant') as MockRV: + with mock.patch('ide.tasks.git.ResourceVariant.VARIANT_STRINGS', {}): + MockRF.objects.create.return_value = mock_resource + MockRV.objects.filter.return_value.first.return_value = None + MockRV.objects.create.return_value = mock_variant + + _upsert_resource_variant(project, repo, change, 'resources/images/icon.png', existing_resources, tag_map, media_map) + + MockRF.objects.create.assert_called_once() + self.assertEqual(MockRF.objects.create.call_args[1]['kind'], 'bitmap') + + def test_infers_kind_when_media_map_is_none(self): + project = mock.MagicMock() + project.resources_path = 'resources' + repo = mock.MagicMock() + + change = mock.MagicMock() + change.filename = 'resources/images/icon.png' + change.sha = 'abc123' + + existing_resources = {} + tag_map = {} + + contents = mock.MagicMock() + contents.encoding = None + contents.decoded_content = b'\x89PNG' + repo.get_contents.return_value = contents + + mock_resource = mock.MagicMock() + mock_variant = mock.MagicMock() + + with mock.patch('ide.tasks.git.ResourceFile') as MockRF: + with mock.patch('ide.tasks.git.ResourceVariant') as MockRV: + with mock.patch('ide.tasks.git.ResourceVariant.VARIANT_STRINGS', {}): + MockRF.objects.create.return_value = mock_resource + MockRV.objects.filter.return_value.first.return_value = None + MockRV.objects.create.return_value = mock_variant + + _upsert_resource_variant(project, repo, change, 'resources/images/icon.png', existing_resources, tag_map) + + MockRF.objects.create.assert_called_once() + self.assertEqual(MockRF.objects.create.call_args[1]['kind'], 'png') + + +class SyncResourceFilesFromManifestTest(TestCase): + @mock.patch('ide.tasks.git.ResourceIdentifier') + def test_corrects_kind_on_existing_resource(self, MockRI): + from ide.tasks.git import _sync_resource_files_from_manifest + project = mock.MagicMock() + project.project_type = 'native' + + existing_resource = mock.MagicMock() + existing_resource.kind = 'png' + existing_resource.is_menu_icon = False + existing_resources = {'images/icon.png': existing_resource} + + media_map = [{'file': 'images/icon.png', 'name': 'ICON', 'type': 'bitmap', 'menuIcon': True}] + + _sync_resource_files_from_manifest(project, media_map, existing_resources) + + self.assertEqual(existing_resource.kind, 'bitmap') + self.assertTrue(existing_resource.is_menu_icon) + existing_resource.save.assert_called_once() + + @mock.patch('ide.tasks.git.ResourceIdentifier') + def test_does_not_save_when_kind_already_correct(self, MockRI): + from ide.tasks.git import _sync_resource_files_from_manifest + project = mock.MagicMock() + project.project_type = 'native' + + existing_resource = mock.MagicMock() + existing_resource.kind = 'png' + existing_resource.is_menu_icon = False + existing_resources = {'images/icon.png': existing_resource} + + media_map = [{'file': 'images/icon.png', 'name': 'ICON', 'type': 'png', 'menuIcon': False}] + + _sync_resource_files_from_manifest(project, media_map, existing_resources) + + self.assertEqual(existing_resource.kind, 'png') + existing_resource.save.assert_not_called() + + +class GithubPullDeltaAtomicityTest(TestCase): + """Verifies that github_last_commit and github_last_sync are persisted in + the same atomic transaction as the file-content changes, so a partial + failure (or a worker kill) can never leave the project with new files but + the old SHA, which would cause the next pull to re-apply the same delta. + """ + + def setUp(self): + self.user = mock.MagicMock() + self.project = mock.MagicMock() + self.project.github_last_commit = 'oldsha' + self.project.resources_path = 'resources' + self.project.project_type = 'native' + self.repo = mock.MagicMock() + + @mock.patch('ide.tasks.git._apply_delta_changes') + @mock.patch('ide.tasks.git.validate_resources_against_tree') + @mock.patch('ide.tasks.git.parse_manifest_from_tree') + @mock.patch('ide.tasks.git.get_root_path') + @mock.patch('ide.tasks.git.now') + def test_delta_pull_passes_new_commit_sha_to_apply_delta_changes( + self, mock_now, mock_get_root, mock_parse, mock_validate, mock_apply): + """The new SHA must be passed through so _apply_delta_changes can + stamp it inside the same atomic block as the file mutations.""" + mock_now.return_value = '2025-01-01T00:00:00Z' + comparison = mock.MagicMock() + comparison.ahead_by = 3 + comparison.files = [mock.MagicMock(filename='src/main.c', status='modified')] + self.repo.compare.return_value = comparison + + mock_commit = mock.MagicMock() + mock_commit.tree.sha = 'treesha' + self.repo.get_git_commit.return_value = mock_commit + mock_tree = mock.MagicMock() + mock_tree.tree = [] + self.repo.get_git_tree.return_value = mock_tree + + mock_parse.return_value = ('', {'projectType': 'native'}) + mock_get_root.return_value = 'src/main.c' + + _github_pull_delta(self.user, self.project, self.repo, 'newsha') + + mock_apply.assert_called_once() + args, kwargs = mock_apply.call_args + self.assertEqual(args[5], 'newsha') + + @mock.patch('ide.tasks.git._sync_resource_files_from_manifest') + @mock.patch('ide.tasks.git.load_manifest_dict') + def test_apply_delta_changes_updates_github_last_commit_inside_atomic( + self, mock_load, mock_sync): + """When called with new_commit_sha, the SHA and last_sync must be + assigned and saved *inside* the transaction.atomic() block.""" + mock_load.return_value = ({}, {}, {}) + project = mock.MagicMock() + project.resources_path = 'resources' + project.project_type = 'native' + project.github_last_commit = 'oldsha' + repo = mock.MagicMock() + + manifest = {'projectType': 'native', 'resources': {'media': []}} + + # Use a context-manager spy that records the project state at __enter__ + # and __exit__ so we can prove the SHA was set before the block exited. + captured = {} + + class SpyAtomic: + def __enter__(self_inner): + captured['enter_github_last_commit'] = project.github_last_commit + return self_inner + + def __exit__(self_inner, *args): + captured['exit_github_last_commit'] = project.github_last_commit + return False + + with mock.patch('ide.tasks.git.transaction.atomic', return_value=SpyAtomic()): + project.source_files.all.return_value = [] + project.resources.all.return_value = [] + _apply_delta_changes(project, repo, '', manifest, [], new_commit_sha='newsha') + + self.assertEqual(captured['enter_github_last_commit'], 'oldsha') + self.assertEqual(captured['exit_github_last_commit'], 'newsha') + self.assertIsNotNone(project.github_last_sync) + + @mock.patch('ide.tasks.git._sync_resource_files_from_manifest') + @mock.patch('ide.tasks.git.load_manifest_dict') + def test_apply_delta_changes_does_not_touch_sha_when_omitted( + self, mock_load, mock_sync): + """When new_commit_sha is None (legacy call sites), the SHA must + not be silently overwritten.""" + mock_load.return_value = ({}, {}, {}) + project = mock.MagicMock() + project.resources_path = 'resources' + project.project_type = 'native' + project.github_last_commit = 'oldsha' + repo = mock.MagicMock() + + manifest = {'projectType': 'native', 'resources': {'media': []}} + + with mock.patch('ide.tasks.git.transaction'): + project.source_files.all.return_value = [] + project.resources.all.return_value = [] + _apply_delta_changes(project, repo, '', manifest, []) + + self.assertEqual(project.github_last_commit, 'oldsha') + + @mock.patch('ide.tasks.git._apply_delta_changes') + @mock.patch('ide.tasks.git.validate_resources_against_tree') + @mock.patch('ide.tasks.git.parse_manifest_from_tree') + @mock.patch('ide.tasks.git.get_root_path') + @mock.patch('ide.tasks.git.now') + @mock.patch('ide.tasks.git.transaction.atomic') + def test_delta_pull_ahead_by_zero_saves_inside_atomic( + self, mock_atomic, mock_now, mock_get_root, mock_parse, mock_validate, mock_apply): + """The 'no new commits' branch must also wrap its save in atomic().""" + mock_atomic.return_value.__enter__ = mock.MagicMock() + mock_atomic.return_value.__exit__ = mock.MagicMock(return_value=False) + mock_now.return_value = '2025-01-01T00:00:00Z' + comparison = mock.MagicMock() + comparison.ahead_by = 0 + self.repo.compare.return_value = comparison + + _github_pull_delta(self.user, self.project, self.repo, 'newsha') + + mock_atomic.assert_called_once() + self.assertEqual(self.project.github_last_commit, 'newsha') + + +class GithubPullFullAtomicityTest(TestCase): + """Verifies that _github_pull_full stamps the new SHA in an atomic block + after do_import_archive, so a project never ends up with new files but + the old github_last_commit. + """ + + def setUp(self): + self.user = mock.MagicMock() + self.project = mock.MagicMock() + self.project.github_last_commit = 'oldsha' + self.project.resources_path = 'resources' + self.project.project_type = 'native' + self.repo = mock.MagicMock() + + @mock.patch('ide.tasks.git.do_import_archive') + @mock.patch('ide.tasks.git.validate_resources_against_tree') + @mock.patch('ide.tasks.git.parse_manifest_from_tree') + @mock.patch('ide.tasks.git.get_root_path') + @mock.patch('ide.tasks.git.urlopen') + @mock.patch('ide.tasks.git.now') + @mock.patch('ide.tasks.git.transaction.atomic') + def test_full_pull_stamps_sha_in_atomic_block( + self, mock_atomic, mock_now, mock_urlopen, mock_get_root, mock_parse, mock_validate, mock_import): + mock_atomic.return_value.__enter__ = mock.MagicMock() + mock_atomic.return_value.__exit__ = mock.MagicMock(return_value=False) + mock_now.return_value = '2025-01-01T00:00:00Z' + + branch = mock.MagicMock() + branch.commit.sha = 'newsha' + self.repo.default_branch = 'main' + + commit = mock.MagicMock() + commit.tree.sha = 'treesha' + self.repo.get_git_commit.return_value = commit + tree = mock.MagicMock() + tree.tree = [] + self.repo.get_git_tree.return_value = tree + + mock_parse.return_value = ('', {'projectType': 'native'}) + mock_get_root.return_value = 'src/main.c' + mock_import.return_value = 'import_result' + + result = _github_pull_full(self.user, self.project, self.repo, branch) + + self.assertEqual(result, 'import_result') + mock_atomic.assert_called_once() + self.assertEqual(self.project.github_last_commit, 'newsha') + self.assertEqual(self.project.github_last_sync, '2025-01-01T00:00:00Z') + self.project.save.assert_called_once() \ No newline at end of file diff --git a/cloudpebble/ide/tests/test_import_archive.skip b/cloudpebble/ide/tests/test_import_archive.py similarity index 71% rename from cloudpebble/ide/tests/test_import_archive.skip rename to cloudpebble/ide/tests/test_import_archive.py index 1b0dbfb..e13e88a 100644 --- a/cloudpebble/ide/tests/test_import_archive.skip +++ b/cloudpebble/ide/tests/test_import_archive.py @@ -246,3 +246,101 @@ def test_import_library_with_resources(self): do_import_archive(self.project_id, bundle) project = Project.objects.get(pk=self.project_id) self.assertSetEqual({f.file_name for f in project.resources.all()}, {'res1.png', 'res2.png'}) + + +@mock.patch('ide.models.s3file.s3', fake_s3) +class TestWipeExisting(CloudpebbleTestCase): + def setUp(self): + self.login() + + def test_wipe_existing_replaces_source_files(self): + """Importing with wipe_existing=True replaces existing source files""" + first_bundle = build_bundle({ + 'src/main.c': '// original', + 'src/utils.c': '// utils', + 'appinfo.json': make_appinfo() + }) + do_import_archive(self.project_id, first_bundle) + project = Project.objects.get(pk=self.project_id) + self.assertEqual(project.source_files.count(), 2) + + second_bundle = build_bundle({ + 'src/replaced.c': '// replaced', + 'appinfo.json': make_appinfo() + }) + do_import_archive(self.project_id, second_bundle, wipe_existing=True) + project = Project.objects.get(pk=self.project_id) + self.assertEqual(project.source_files.count(), 1) + self.assertEqual(project.source_files.first().file_name, 'replaced.c') + + def test_wipe_existing_replaces_resources(self): + """Importing with wipe_existing=True replaces existing resources""" + first_bundle = build_bundle({ + 'src/main.c': '', + 'resources/images/blah.png': 'original!', + 'appinfo.json': make_appinfo(options={ + 'resources': { + 'media': [{ + 'file': 'images/blah.png', + 'name': 'IMAGE_BLAH', + 'type': 'bitmap' + }] + } + }) + }) + do_import_archive(self.project_id, first_bundle) + project = Project.objects.get(pk=self.project_id) + self.assertEqual(project.resources.count(), 1) + + second_bundle = build_bundle({ + 'src/main.c': '', + 'resources/images/new.png': 'new!', + 'appinfo.json': make_appinfo(options={ + 'resources': { + 'media': [{ + 'file': 'images/new.png', + 'name': 'IMAGE_NEW', + 'type': 'bitmap' + }] + } + }) + }) + do_import_archive(self.project_id, second_bundle, wipe_existing=True) + project = Project.objects.get(pk=self.project_id) + self.assertEqual(project.resources.count(), 1) + self.assertEqual(project.resources.first().file_name, 'new.png') + + def test_wipe_existing_false_keeps_files(self): + """Importing with wipe_existing=False (default) does not delete existing files""" + first_bundle = build_bundle({ + 'src/main.c': '// original', + 'appinfo.json': make_appinfo() + }) + do_import_archive(self.project_id, first_bundle) + project = Project.objects.get(pk=self.project_id) + self.assertEqual(project.source_files.count(), 1) + + second_bundle = build_bundle({ + 'src/added.c': '// added', + 'appinfo.json': make_appinfo() + }) + do_import_archive(self.project_id, second_bundle, wipe_existing=False) + project = Project.objects.get(pk=self.project_id) + self.assertEqual(project.source_files.count(), 2) + + def test_wipe_existing_rolls_back_on_failure(self): + """If wipe_existing=True and import fails, original files are preserved""" + first_bundle = build_bundle({ + 'src/main.c': '// original', + 'appinfo.json': make_appinfo() + }) + do_import_archive(self.project_id, first_bundle) + project = Project.objects.get(pk=self.project_id) + self.assertEqual(project.source_files.count(), 1) + + bad_bundle = b'this is not a valid zip file' + with self.assertRaises(Exception): + do_import_archive(self.project_id, bad_bundle, wipe_existing=True) + project = Project.objects.get(pk=self.project_id) + self.assertEqual(project.source_files.count(), 1, + "Original files should be preserved after failed import") diff --git a/cloudpebble/ide/tests/test_sse.py b/cloudpebble/ide/tests/test_sse.py new file mode 100644 index 0000000..01c5ceb --- /dev/null +++ b/cloudpebble/ide/tests/test_sse.py @@ -0,0 +1,382 @@ +import json +from io import BytesIO +from unittest import mock + +from django.test import TestCase, RequestFactory +from django.contrib.auth.models import User + +from utils.events import publish_event +from ide.api.sse import SSEEventStream, project_events +from ide.tasks.git import hooked_commit, do_github_pull +from ide.tasks.build import run_compile + + +class FakePubSub: + def __init__(self): + self.messages = [] + self.subscribed = False + + def subscribe(self, channel): + self.subscribed = True + + def listen(self): + for msg in self.messages: + yield msg + + def unsubscribe(self, channel): + self.subscribed = False + + def close(self): + pass + + +class FakeRedisClient: + def __init__(self): + self.published = [] + self._pubsub = FakePubSub() + + def publish(self, channel, message): + self.published.append((channel, message)) + + def pubsub(self): + return self._pubsub + + +class TestPublishEvent(TestCase): + @mock.patch('utils.events.redis_client') + def test_publish_event_sends_json(self, mock_redis): + publish_event(42, 'pull_start') + mock_redis.publish.assert_called_once_with( + 'project_events:42', + json.dumps({'type': 'pull_start'}) + ) + + @mock.patch('utils.events.redis_client') + def test_publish_event_includes_kwargs(self, mock_redis): + publish_event(7, 'build_complete', build_id=99, state='succeeded') + channel, message = mock_redis.publish.call_args[0] + self.assertEqual(channel, 'project_events:7') + data = json.loads(message) + self.assertEqual(data['type'], 'build_complete') + self.assertEqual(data['build_id'], 99) + self.assertEqual(data['state'], 'succeeded') + + @mock.patch('utils.events.redis_client') + def test_publish_event_survives_redis_error(self, mock_redis): + import redis as redis_lib + mock_redis.publish.side_effect = redis_lib.ConnectionError("Connection refused") + publish_event(42, 'pull_start') + mock_redis.publish.assert_called_once() + + @mock.patch('utils.events.redis_client') + def test_publish_event_survives_generic_redis_error(self, mock_redis): + import redis as redis_lib + mock_redis.publish.side_effect = redis_lib.RedisError("timeout") + publish_event(42, 'pull_start') + mock_redis.publish.assert_called_once() + + +class TestSSEEventStream(TestCase): + def test_stream_yields_formatted_messages(self): + stream = SSEEventStream.__new__(SSEEventStream) + stream.channel = 'project_events:1' + stream.pubsub = FakePubSub() + stream.pubsub.messages = [ + {'type': 'subscribe', 'data': b''}, + {'type': 'message', 'data': b'{"type":"pull_start"}'}, + {'type': 'message', 'data': b'{"type":"pull_complete","github_last_commit":"abc123","github_last_sync":"2025-01-01"}'}, + ] + results = [] + for item in stream: + results.append(item) + self.assertEqual(len(results), 2) + self.assertEqual(results[0], 'event: pull_start\ndata: {}\n\n') + self.assertEqual(results[1], 'event: pull_complete\ndata: {"github_last_commit": "abc123", "github_last_sync": "2025-01-01"}\n\n') + + def test_stream_skips_non_message_types(self): + stream = SSEEventStream.__new__(SSEEventStream) + stream.channel = 'project_events:1' + stream.pubsub = FakePubSub() + stream.pubsub.messages = [ + {'type': 'subscribe', 'data': b''}, + {'type': 'message', 'data': b'{"type":"build_start","build_id":1}'}, + ] + results = [] + for item in stream: + results.append(item) + self.assertEqual(len(results), 1) + self.assertIn('event:', results[0]) + self.assertIn('build_start', results[0]) + + def test_stream_decodes_bytes_data(self): + stream = SSEEventStream.__new__(SSEEventStream) + stream.channel = 'project_events:1' + stream.pubsub = FakePubSub() + stream.pubsub.messages = [ + {'type': 'message', 'data': b'{"type":"pull_start"}'}, + ] + results = list(stream) + self.assertEqual(results[0], 'event: pull_start\ndata: {}\n\n') + + def test_stream_handles_string_data(self): + stream = SSEEventStream.__new__(SSEEventStream) + stream.channel = 'project_events:1' + stream.pubsub = FakePubSub() + stream.pubsub.messages = [ + {'type': 'message', 'data': '{"type":"pull_start"}'}, + ] + results = list(stream) + self.assertEqual(results[0], 'event: pull_start\ndata: {}\n\n') + + def test_stream_cleans_up_on_generator_exit(self): + stream = SSEEventStream.__new__(SSEEventStream) + stream.channel = 'project_events:1' + fake_pubsub = FakePubSub() + stream.pubsub = fake_pubsub + gen = iter(stream) + next(gen, None) + gen.close() + self.assertFalse(fake_pubsub.subscribed) + + +class TestProjectEventsEndpoint(TestCase): + def setUp(self): + self.factory = RequestFactory() + self.user = User.objects.create_user('testuser', 'test@test.test', 'testpass') + + def test_requires_login(self): + from django.test import Client + client = Client() + response = client.get('/ide/project/1/events') + self.assertIn(response.status_code, [301, 302, 403]) + + def test_nonexistent_project_returns_404(self): + request = self.factory.get('/ide/project/99999/events') + request.user = self.user + with mock.patch('ide.api.sse.redis_client') as mock_redis: + mock_redis.pubsub.return_value = FakePubSub() + response = project_events(request, 99999) + self.assertEqual(response.status_code, 404) + + @mock.patch('ide.api.sse.redis_client') + def test_response_headers(self, mock_redis): + from ide.models.project import Project + project = Project.objects.create(owner=self.user, name='testproject') + mock_redis.pubsub.return_value = FakePubSub() + request = self.factory.get('/ide/project/%d/events' % project.id) + request.user = self.user + response = project_events(request, project.id) + self.assertEqual(response['Content-Type'], 'text/event-stream') + self.assertEqual(response['Cache-Control'], 'no-cache') + self.assertEqual(response['X-Accel-Buffering'], 'no') + + +class TestHookedCommitEvents(TestCase): + @mock.patch('ide.tasks.git.publish_event') + @mock.patch('ide.tasks.git.run_compile') + @mock.patch('ide.tasks.git.github_pull') + def test_publishes_pull_start_and_complete(self, mock_pull, mock_compile, mock_publish): + from ide.models.project import Project + user = User.objects.create_user('hooktest', 'hook@test.test', 'testpass') + project = Project.objects.create( + owner=user, name='hookproj', + github_repo='owner/repo', github_branch='main', + github_last_commit='oldsha', github_hook_build=False, + ) + mock_pull.return_value = True + hooked_commit(project.id, 'newsha') + publish_calls = mock_publish.call_args_list + self.assertEqual(publish_calls[0][0], (project.id, 'pull_start')) + self.assertEqual(publish_calls[1][0][0], project.id) + self.assertEqual(publish_calls[1][0][1], 'pull_complete') + self.assertIn('github_last_commit', publish_calls[1][1]) + self.assertIn('github_last_sync', publish_calls[1][1]) + + @mock.patch('ide.tasks.git.publish_event') + @mock.patch('ide.tasks.git.run_compile') + @mock.patch('ide.tasks.git.github_pull') + def test_publishes_build_start_when_auto_build(self, mock_pull, mock_compile, mock_publish): + from ide.models.project import Project + from ide.models.build import BuildResult + user = User.objects.create_user('hooktest2', 'hook2@test.test', 'testpass') + project = Project.objects.create( + owner=user, name='hookproj2', + github_repo='owner/repo', github_branch='main', + github_last_commit='oldsha', github_hook_build=True, + ) + mock_pull.return_value = True + hooked_commit(project.id, 'newsha') + build_start_call = None + for call in mock_publish.call_args_list: + if call[0][1] == 'build_start': + build_start_call = call + break + self.assertIsNotNone(build_start_call) + self.assertEqual(build_start_call[0][0], project.id) + self.assertIn('build_id', build_start_call[1]) + self.assertIsInstance(build_start_call[1]['build_id'], int) + + @mock.patch('ide.tasks.git.publish_event') + @mock.patch('ide.tasks.git.github_pull') + def test_publishes_pull_failed_on_exception(self, mock_pull, mock_publish): + from ide.models.project import Project + user = User.objects.create_user('hooktest3', 'hook3@test.test', 'testpass') + project = Project.objects.create( + owner=user, name='hookproj3', + github_repo='owner/repo', github_branch='main', + github_last_commit='oldsha', github_hook_build=False, + ) + mock_pull.side_effect = Exception('pull failed') + with self.assertRaises(Exception) as ctx: + hooked_commit(project.id, 'newsha') + self.assertEqual(str(ctx.exception), 'pull failed') + types = [call[0][1] for call in mock_publish.call_args_list] + self.assertEqual(types, ['pull_start', 'pull_failed']) + + @mock.patch('ide.tasks.git.publish_event') + @mock.patch('ide.tasks.git.run_compile') + @mock.patch('ide.tasks.git.github_pull') + def test_no_pull_events_when_commit_unchanged(self, mock_pull, mock_compile, mock_publish): + from ide.models.project import Project + user = User.objects.create_user('hooknoch', 'hooknoch@test.test', 'testpass') + project = Project.objects.create( + owner=user, name='hooknoch', + github_repo='owner/repo', github_branch='main', + github_last_commit='samesha', github_hook_build=False, + ) + result = hooked_commit(project.id, 'samesha') + self.assertFalse(result) + mock_publish.assert_not_called() + mock_pull.assert_not_called() + + @mock.patch('ide.tasks.git.publish_event') + @mock.patch('ide.tasks.git.run_compile') + @mock.patch('ide.tasks.git.github_pull') + def test_skip_build_when_auto_build_disabled(self, mock_pull, mock_compile, mock_publish): + from ide.models.project import Project + user = User.objects.create_user('hooknobuild', 'hooknobuild@test.test', 'testpass') + project = Project.objects.create( + owner=user, name='hooknobuild', + github_repo='owner/repo', github_branch='main', + github_last_commit='oldsha', github_hook_build=False, + ) + mock_pull.return_value = True + hooked_commit(project.id, 'newsha') + types = [call[0][1] for call in mock_publish.call_args_list] + self.assertNotIn('build_start', types) + mock_compile.assert_not_called() + + +class TestDoGithubPullEvents(TestCase): + @mock.patch('ide.tasks.git.publish_event') + @mock.patch('ide.tasks.git.run_compile') + @mock.patch('ide.tasks.git.github_pull') + def test_publishes_pull_events(self, mock_pull, mock_compile, mock_publish): + from ide.models.project import Project + user = User.objects.create_user('pulltest1', 'pull1@test.test', 'testpass') + project = Project.objects.create( + owner=user, name='pullproj1', + github_repo='owner/repo', github_branch='main', + github_last_commit='oldsha', github_hook_build=False, + ) + mock_pull.return_value = True + do_github_pull(project.id) + types = [call[0][1] for call in mock_publish.call_args_list] + self.assertEqual(types[0], 'pull_start') + self.assertEqual(types[1], 'pull_complete') + + @mock.patch('ide.tasks.git.publish_event') + @mock.patch('ide.tasks.git.run_compile') + @mock.patch('ide.tasks.git.github_pull') + def test_auto_builds_when_hook_build_enabled(self, mock_pull, mock_compile, mock_publish): + from ide.models.project import Project + user = User.objects.create_user('pulltest2', 'pull2@test.test', 'testpass') + project = Project.objects.create( + owner=user, name='pullproj2', + github_repo='owner/repo', github_branch='main', + github_last_commit='oldsha', github_hook_build=True, + ) + mock_pull.return_value = True + do_github_pull(project.id) + types = [call[0][1] for call in mock_publish.call_args_list] + self.assertIn('build_start', types) + mock_compile.assert_called_once() + + @mock.patch('ide.tasks.git.publish_event') + @mock.patch('ide.tasks.git.run_compile') + @mock.patch('ide.tasks.git.github_pull') + def test_no_auto_build_when_hook_build_disabled(self, mock_pull, mock_compile, mock_publish): + from ide.models.project import Project + user = User.objects.create_user('pulltest3', 'pull3@test.test', 'testpass') + project = Project.objects.create( + owner=user, name='pullproj3', + github_repo='owner/repo', github_branch='main', + github_last_commit='oldsha', github_hook_build=False, + ) + mock_pull.return_value = True + do_github_pull(project.id) + types = [call[0][1] for call in mock_publish.call_args_list] + self.assertNotIn('build_start', types) + mock_compile.assert_not_called() + + @mock.patch('ide.tasks.git.publish_event') + @mock.patch('ide.tasks.git.github_pull') + def test_publishes_pull_failed_on_exception(self, mock_pull, mock_publish): + from ide.models.project import Project + user = User.objects.create_user('pulltest4', 'pull4@test.test', 'testpass') + project = Project.objects.create( + owner=user, name='pullproj4', + github_repo='owner/repo', github_branch='main', + github_last_commit='oldsha', github_hook_build=False, + ) + mock_pull.side_effect = Exception('pull failed') + with self.assertRaises(Exception): + do_github_pull(project.id) + types = [call[0][1] for call in mock_publish.call_args_list] + self.assertEqual(types, ['pull_start', 'pull_failed']) + + @mock.patch('ide.tasks.git.publish_event') + @mock.patch('ide.tasks.git.run_compile') + @mock.patch('ide.tasks.git.github_pull') + def test_no_auto_build_when_nothing_changed(self, mock_pull, mock_compile, mock_publish): + from ide.models.project import Project + user = User.objects.create_user('pulltest5', 'pull5@test.test', 'testpass') + project = Project.objects.create( + owner=user, name='pullproj5', + github_repo='owner/repo', github_branch='main', + github_last_commit='oldsha', github_hook_build=True, + ) + mock_pull.return_value = False + do_github_pull(project.id) + types = [call[0][1] for call in mock_publish.call_args_list] + self.assertNotIn('build_start', types) + mock_compile.assert_not_called() + + +class TestRunCompileEvents(TestCase): + @mock.patch('ide.tasks.build.publish_event') + @mock.patch('ide.tasks.build.assemble_project') + @mock.patch('ide.tasks.build._set_resource_limits') + @mock.patch('ide.tasks.build.shutil.rmtree') + @mock.patch('ide.tasks.build.now') + @mock.patch('ide.tasks.build.os.chdir') + @mock.patch('ide.tasks.build.subprocess.check_output', side_effect=Exception('boom')) + def test_publishes_build_complete_on_failure(self, mock_subprocess, mock_chdir, mock_now, mock_rmtree, mock_limits, mock_assemble, mock_publish): + from ide.models.project import Project + from ide.models.build import BuildResult + user = User.objects.create_user('buildtest', 'build@test.test', 'testpass') + project = Project.objects.create(owner=user, name='buildproj') + build = BuildResult.objects.create(project=project) + mock_now.return_value = build.started + try: + run_compile(build.id) + except Exception: + pass + build.refresh_from_db() + self.assertEqual(build.state, BuildResult.STATE_FAILED) + mock_publish.assert_called_once() + call_args = mock_publish.call_args + self.assertEqual(call_args[0][0], project.id) + self.assertEqual(call_args[0][1], 'build_complete') + self.assertEqual(call_args[1]['build_id'], build.id) + self.assertEqual(call_args[1]['state'], 'failed') \ No newline at end of file diff --git a/cloudpebble/ide/urls.py b/cloudpebble/ide/urls.py index 435df19..1024c6f 100644 --- a/cloudpebble/ide/urls.py +++ b/cloudpebble/ide/urls.py @@ -3,6 +3,7 @@ from ide.api import proxy_keen, check_task, get_shortlink, heartbeat from ide.api.git import github_push, github_pull, set_project_repo, create_project_repo +from ide.api.sse import project_events from ide.api.phone import ping_phone, check_phone, list_phones, update_phone from ide.api.project import ( project_info, @@ -243,6 +244,11 @@ github_hook, name="github_hook", ), + re_path( + r"^project/(?P\d+)/events$", + project_events, + name="project_events", + ), re_path( r"^project/(?P\d+)/status\.png$", build_status, name="build_status" ), diff --git a/cloudpebble/ide/utils/cloudpebble_test.py b/cloudpebble/ide/utils/cloudpebble_test.py index 9b7060e..0b57033 100644 --- a/cloudpebble/ide/utils/cloudpebble_test.py +++ b/cloudpebble/ide/utils/cloudpebble_test.py @@ -10,7 +10,6 @@ from ide.utils.sdk import dict_to_pretty_json from django.test import TestCase from django.test.client import Client -from django.test.utils import setup_test_environment from ide.models.user import User try: @@ -18,8 +17,6 @@ except ImportError: from django.test.utils import override_settings -setup_test_environment() - # TODO: after moving to Django 1.9, use client.post().json() instead of json.loads(client.post().content) diff --git a/cloudpebble/utils/events.py b/cloudpebble/utils/events.py new file mode 100644 index 0000000..47e212f --- /dev/null +++ b/cloudpebble/utils/events.py @@ -0,0 +1,19 @@ +import json +import logging + +import redis as redis_lib +from utils.redis_helper import redis_client + +logger = logging.getLogger(__name__) + + +def publish_event(project_id, event_type, **kwargs): + data = {'type': event_type} + data.update(kwargs) + channel = 'project_events:{}'.format(project_id) + try: + redis_client.publish(channel, json.dumps(data)) + except (redis_lib.RedisError, redis_lib.ConnectionError, ConnectionError): + logger.warning("Failed to publish %s to %s (Redis unavailable)", event_type, channel, exc_info=True) + return + logger.debug("Published %s to %s", event_type, channel) \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 627d391..c53d2ab 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,6 +18,9 @@ services: args: PEBBLE_SDK_VERSION: ${PEBBLE_SDK_VERSION} NODE_VERSION: ${NODE_VERSION_WEB:-20.11.0} + env_file: + - path: .env.local + required: false links: - redis - postgres @@ -61,6 +64,9 @@ services: args: PEBBLE_SDK_VERSION: ${PEBBLE_SDK_VERSION} NODE_VERSION: ${NODE_VERSION_WEB:-20.11.0} + env_file: + - path: .env.local + required: false links: - redis - postgres diff --git a/nginx/nginx.conf b/nginx/nginx.conf index ad6f2f1..314d189 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -43,5 +43,17 @@ http { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto https; } + + location ~ ^/ide/project/\d+/events$ { + proxy_pass http://web:80; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 86400; + chunked_transfer_encoding off; + } } } diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 0000000..7cb777f --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,10 @@ +{ + "executionEnvironments": [ + { + "root": "cloudpebble" + } + ], + "extraPaths": [ + "cloudpebble" + ] +} \ No newline at end of file