diff --git a/packages/app/package.json b/packages/app/package.json index ccc1b69f84c..c561e3cf7f9 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -107,7 +107,7 @@ "circular-json": "^0.4.0", "codemirror": "^5.27.4", "codesandbox-api": "0.0.24", - "codesandbox-import-utils": "^2.1.14", + "codesandbox-import-utils": "^2.2.1", "color": "^0.11.4", "compare-versions": "^3.1.0", "console": "^0.7.2", diff --git a/packages/app/src/app/overmind/effects/api/index.ts b/packages/app/src/app/overmind/effects/api/index.ts index d9538d2e7f8..2d0e44fbe7d 100755 --- a/packages/app/src/app/overmind/effects/api/index.ts +++ b/packages/app/src/app/overmind/effects/api/index.ts @@ -139,6 +139,24 @@ export default { ) .then(transformModule); }, + saveModulePrivateUpload( + sandboxId: string, + moduleShortid: string, + data: { + code: string; + uploadId: string; + sha: string; + } + ): Promise { + return api + .put( + `/sandboxes/${sandboxId}/modules/${moduleShortid}`, + { + module: data, + } + ) + .then(transformModule); + }, saveModules(sandboxId: string, modules: Module[]): Promise { return api .put(`/sandboxes/${sandboxId}/modules/mupdate`, { diff --git a/packages/app/src/app/overmind/effects/http.ts b/packages/app/src/app/overmind/effects/http.ts index 0330211dbc8..114ec68a739 100755 --- a/packages/app/src/app/overmind/effects/http.ts +++ b/packages/app/src/app/overmind/effects/http.ts @@ -7,4 +7,24 @@ export default { delete: axios.delete, put: axios.put, request: axios.request, + blobToBase64: (url: string): Promise => + fetch(url) + .then((response) => response.blob()) + .then( + (blob) => + new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = function () { + // Github interprets base64 differently, so this fixes it, insane right? + // https://stackoverflow.com/questions/39234218/github-api-upload-an-image-to-repo-from-base64-array?rq=1 + resolve( + window.btoa( + window.atob((reader.result as string).replace(/^(.+,)/, '')) + ) + ); + }; + reader.onerror = reject; + reader.readAsDataURL(blob); + }) + ), }; diff --git a/packages/app/src/app/overmind/namespaces/files/actions.ts b/packages/app/src/app/overmind/namespaces/files/actions.ts index 0275f7f1713..d735d0eeefa 100755 --- a/packages/app/src/app/overmind/namespaces/files/actions.ts +++ b/packages/app/src/app/overmind/namespaces/files/actions.ts @@ -1,7 +1,7 @@ import { getDirectoryPath, getModulePath, - getModulesAndDirectoriesInDirectory, + getModulesAndDirectoriesInDirectory } from '@codesandbox/common/lib/sandbox/modules'; import getDefinition from '@codesandbox/common/lib/templates'; import { Directory, Module, UploadFile } from '@codesandbox/common/lib/types'; @@ -15,7 +15,7 @@ import denormalize from 'codesandbox-import-utils/lib/utils/files/denormalize'; import { resolveDirectoryWrapped, - resolveModuleWrapped, + resolveModuleWrapped } from '../../utils/resolve-module-wrapped'; import * as internalActions from './internalActions'; @@ -33,13 +33,13 @@ export const applyRecover: Action { actions.editor.codeChanged({ moduleShortid: module.shortid, - code: recoverData.code, + code: recoverData.code }); effects.vscode.setModuleCode(module); }); effects.analytics.track('Files Recovered', { - fileCount: recoveredList.length, + fileCount: recoveredList.length }); }; @@ -56,13 +56,13 @@ export const createRecoverDiffs: Action = async ( } catch (error) { actions.internal.handleError({ message: 'Unable to get uploaded files information', - error, + error }); } }; @@ -491,7 +491,7 @@ export const addedFileToSandbox: AsyncAction = async ( state.uploadedFiles.splice(index, 0, ...removedFiles); actions.internal.handleError({ message: 'Unable to delete uploaded file', - error, + error }); } }; @@ -544,28 +544,29 @@ export const filesUploaded: AsyncAction<{ const { modules, directories } = await actions.files.internal.uploadFiles( { files, - directoryShortid, + directoryShortid } ); actions.files.massCreateModules({ modules, directories, - directoryShortid, + directoryShortid }); effects.executor.updateFiles(sandbox); + actions.git.updateGitChanges(); } catch (error) { if (error.message.indexOf('413') !== -1) { actions.internal.handleError({ message: `The uploaded file is bigger than 7MB, contact hello@codesandbox.io if you want to raise this limit`, error, - hideErrorMessage: true, + hideErrorMessage: true }); } else { actions.internal.handleError({ message: 'Unable to upload files', - error, + error }); } } @@ -631,7 +632,7 @@ export const massCreateModules: AsyncAction<{ actions.internal.handleError({ message: 'Unable to create new files', - error, + error }); } }, @@ -662,7 +663,7 @@ export const moduleCreated: AsyncAction<{ sourceId: sandbox.sourceId, isNotSynced: true, ...(code ? { code } : {}), - ...(typeof isBinary === 'boolean' ? { isBinary } : {}), + ...(typeof isBinary === 'boolean' ? { isBinary } : {}) }); // We have to push the module to the array before we can figure out its path, @@ -733,7 +734,7 @@ export const moduleCreated: AsyncAction<{ actions.internal.handleError({ message: 'Unable to save new file', - error, + error }); } @@ -806,7 +807,7 @@ export const createModulesByPath: AsyncAction<{ modules, directories, directoryShortid: null, - cbID, + cbID }); effects.executor.updateFiles(sandbox); @@ -880,7 +881,7 @@ export const syncSandbox: AsyncAction = async ( actions.internal.handleError({ message: "We weren't able to retrieve the latest files of the sandbox, please refresh", - error, + error }); } diff --git a/packages/app/src/app/overmind/namespaces/git/actions.ts b/packages/app/src/app/overmind/namespaces/git/actions.ts index 1a35c4b8101..17406935e50 100755 --- a/packages/app/src/app/overmind/namespaces/git/actions.ts +++ b/packages/app/src/app/overmind/namespaces/git/actions.ts @@ -166,9 +166,9 @@ export const createRepoClicked: AsyncAction = async ({ state.currentModal = null; actions.editor.internal.forkSandbox({ - sandboxId: `github/${git.username}/${git.repo}/tree/${ - git.branch - }/${git.path || ''}`, + sandboxId: `github/${git.username}/${git.repo}/tree/${git.branch}/${ + git.path || '' + }`, }); } catch (error) { actions.internal.handleError({ @@ -245,7 +245,7 @@ export const createCommitClicked: AsyncAction = async ({ } } - const changes = actions.git._getGitChanges(); + const changes = await actions.git._getGitChanges(); const commit = await effects.api.createGitCommit( sandbox.id, `${git.title}\n${git.description}`, @@ -256,18 +256,13 @@ export const createCommitClicked: AsyncAction = async ({ ? [git.sourceCommitSha!, git.baseCommitSha] : [git.sourceCommitSha!] ); - changes.added.forEach(change => { - git.sourceModulesByPath[change.path] = change.content; - }); - changes.modified.forEach(change => { - git.sourceModulesByPath[change.path] = change.content; - }); - changes.deleted.forEach(path => { - delete git.sourceModulesByPath[path]; - }); - actions.git._setGitChanges(); + + // We need to load the source again as it has now changed. We can not optimistically deal with + // this, cause you might have added a binary sandbox.originalGit!.commitSha = commit.sha; sandbox.originalGitCommitSha = commit.sha; + await actions.git._loadSourceSandbox(); + actions.git._setGitChanges(); state.git.isCommitting = false; state.git.title = ''; state.git.description = ''; @@ -335,7 +330,7 @@ export const createPrClicked: AsyncAction = async ({ } } - const changes = actions.git._getGitChanges(); + const changes = await actions.git._getGitChanges(); const pr = await effects.api.createGitPr( id, state.git.title, @@ -344,10 +339,10 @@ export const createPrClicked: AsyncAction = async ({ ); changes.added.forEach(change => { - git.sourceModulesByPath[change.path] = change.content; + git.sourceModulesByPath[change.path].code = change.content; }); changes.modified.forEach(change => { - git.sourceModulesByPath[change.path] = change.content; + git.sourceModulesByPath[change.path].code = change.content; }); changes.deleted.forEach(path => { delete git.sourceModulesByPath[path]; @@ -438,7 +433,9 @@ export const addConflictedFile: AsyncAction = async ( [conflict.filename]: { content: conflict.content!, isBinary: false }, }, }); - state.git.sourceModulesByPath['/' + conflict.filename] = conflict.content!; + state.git.sourceModulesByPath[ + '/' + conflict.filename + ].code = conflict.content!; state.git.conflictsResolving.splice( state.git.conflictsResolving.indexOf(conflict.filename), @@ -499,71 +496,25 @@ export const resolveOutOfSync: AsyncAction = async ({ effects.analytics.track('GitHub - Resolve out of sync'); const git = state.git; const { added, deleted, modified } = git.outOfSyncUpdates; - git.isResolving = true; - if (added.length) { - await actions.files.createModulesByPath({ - files: added.reduce((aggr, change) => { - aggr[change.filename] = { content: change.content }; - - return aggr; - }, {}), - }); - // We optimistically keep source in sync - added.forEach(change => { - git.sourceModulesByPath['/' + change.filename] = change.content!; - }); - } - - if (deleted.length) { - await Promise.all( - deleted.map(change => { - const module = state.editor.modulesByPath['/' + change.filename]; - return actions.files.moduleDeleted({ moduleShortid: module.shortid }); - }) - ); - // We optimistically keep source in sync - deleted.forEach(change => { - delete git.sourceModulesByPath['/' + change.filename]; - }); - } - if (modified.length) { - await Promise.all( - modified.map(change => { - const module = state.editor.modulesByPath['/' + change.filename]; - - actions.editor.setCode({ - moduleShortid: module.shortid, - code: change.content!, - }); - return actions.editor.codeSaved({ - moduleShortid: module.shortid, - code: change.content!, - cbID: null, - }); - }) - ); - // We optimistically keep source in sync - modified.forEach(change => { - git.sourceModulesByPath['/' + change.filename] = change.content!; - }); - } + git.isResolving = true; const sandbox = state.editor.currentSandbox!; - // When we have a PR and the source is out of sync with base, we need to create a commit to update it + // When we have a PR and the source is out of sync with base, we need to create a commit to update it. We do this + // first, because we need the new source to deal with binary files if (git.gitState === SandboxGitState.OUT_OF_SYNC_PR_BASE) { const changes: GitChanges = { added: added.map(change => ({ path: '/' + change.filename, content: change.content!, - encoding: 'utf-8', + encoding: change.isBinary ? 'base64' : 'utf-8', })), deleted: deleted.map(change => '/' + change.filename), modified: modified.map(change => ({ path: '/' + change.filename, content: change.content!, - encoding: 'utf-8', + encoding: change.isBinary ? 'base64' : 'utf-8', })), }; const commit = await effects.api.createGitCommit( @@ -588,6 +539,77 @@ export const resolveOutOfSync: AsyncAction = async ({ await actions.git._loadSourceSandbox(); + if (added.length) { + await actions.files.createModulesByPath({ + files: added.reduce((aggr, change) => { + aggr[change.filename] = change.isBinary + ? { + content: git.sourceModulesByPath['/' + change.filename].code, + isBinary: true, + uploadId: git.sourceModulesByPath['/' + change.filename].uploadId, + sha: git.sourceModulesByPath['/' + change.filename].sha, + } + : { content: change.content }; + + return aggr; + }, {}), + }); + } + + if (deleted.length) { + await Promise.all( + deleted.map(change => { + const module = state.editor.modulesByPath['/' + change.filename]; + + return actions.files.moduleDeleted({ moduleShortid: module.shortid }); + }) + ); + } + if (modified.length) { + await Promise.all( + modified.map(change => { + const module = state.editor.modulesByPath['/' + change.filename]; + + // If we are dealing with a private binary change, we need to bluntly update + // the module + if (git.sourceModulesByPath['/' + change.filename].sha) { + const code = git.sourceModulesByPath['/' + change.filename].code; + const uploadId = git.sourceModulesByPath['/' + change.filename] + .uploadId!; + const sha = git.sourceModulesByPath['/' + change.filename].sha!; + + const sandboxModule = sandbox.modules.find( + moduleItem => moduleItem.shortid === module.shortid + )!; + sandboxModule.code = code; + sandboxModule.uploadId = uploadId; + sandboxModule.sha = sha; + + return effects.api + .saveModulePrivateUpload(sandbox.id, module.shortid, { + code, + uploadId, + sha, + }) + .then(() => {}); + } + actions.editor.setCode({ + moduleShortid: module.shortid, + code: change.isBinary + ? git.sourceModulesByPath['/' + change.filename].code + : change.content!, + }); + return actions.editor.codeSaved({ + moduleShortid: module.shortid, + code: change.isBinary + ? git.sourceModulesByPath['/' + change.filename].code + : change.content!, + cbID: null, + }); + }) + ); + } + actions.git._setGitChanges(); git.outOfSyncUpdates.added = []; git.outOfSyncUpdates.deleted = []; @@ -611,8 +633,10 @@ export const _setGitChanges: Action = ({ state }) => { if (!(module.path in state.git.sourceModulesByPath)) { changes.added.push(module.path); } else if ( - !module.isBinary && - state.git.sourceModulesByPath[module.path] !== module.code + (module.sha && + state.git.sourceModulesByPath[module.path].sha !== module.sha) || + (!module.sha && + state.git.sourceModulesByPath[module.path].code !== module.code) ) { changes.modified.push(module.path); } @@ -640,6 +664,7 @@ export const _evaluateGitChanges: AsyncAction< if ( change.status === 'removed' && state.editor.modulesByPath[path] && + !(state.editor.modulesByPath[path] as Module).isBinary && (state.editor.modulesByPath[path] as Module).code !== change.content ) { return aggr.concat(change); @@ -654,9 +679,10 @@ export const _evaluateGitChanges: AsyncAction< // We are in conflict if the source changed the file and sandbox also changed the file if ( change.status === 'modified' && + !(state.editor.modulesByPath[path] as Module).isBinary && (state.editor.modulesByPath[path] as Module).code !== change.content && (state.editor.modulesByPath[path] as Module).code !== - state.git.sourceModulesByPath[path] + state.git.sourceModulesByPath[path].code ) { return aggr.concat(change); } @@ -732,7 +758,12 @@ export const _loadSourceSandbox: AsyncAction = async ({ state, effects }) => { ); module.path = path; if (path) { - aggr[path] = module.code; + aggr[path] = { + code: module.code, + isBinary: module.isBinary, + uploadId: module.uploadId, + sha: module.sha, + }; } return aggr; @@ -859,28 +890,43 @@ export const _compareWithBase: AsyncAction = async ({ } }; -export const _getGitChanges: Action = ({ state }) => { +export const _getGitChanges: AsyncAction = async ({ + state, + effects, +}) => { const git = state.git; const sandbox = state.editor.currentSandbox!; return { - added: git.gitChanges.added.map(path => { - const module = sandbox.modules.find( - moduleItem => moduleItem.path === path - ); + added: await Promise.all( + git.gitChanges.added.map(async path => { + const module = sandbox.modules.find( + moduleItem => moduleItem.path === path + ); - return { - path, - content: module!.code, - encoding: 'utf-8', - }; - }), + if (module!.isBinary) { + return { + path, + content: await effects.http.blobToBase64(module!.code), + encoding: 'base64' as 'base64', + }; + } + + return { + path, + content: module!.code, + encoding: 'utf-8' as 'utf-8', + }; + }) + ), deleted: git.gitChanges.deleted, modified: git.gitChanges.modified.map(path => { const module = sandbox.modules.find( moduleItem => moduleItem.path === path ); + // A binary can not be modified, because we have no mechanism for comparing + // private binary files, as their urls are based on moduleId (which is different across sandboxes) return { path, content: module!.code, @@ -914,7 +960,7 @@ export const _tryResolveConflict: AsyncAction = async ({ ) { state.git.isCommitting = true; const sandbox = state.editor.currentSandbox!; - const changes = actions.git._getGitChanges(); + const changes = await actions.git._getGitChanges(); state.git.title = 'Resolve conflict'; const commit = await effects.api.createGitCommit( sandbox.id, diff --git a/packages/app/src/app/overmind/namespaces/git/state.ts b/packages/app/src/app/overmind/namespaces/git/state.ts index cd25f8a7236..afc3160d79f 100755 --- a/packages/app/src/app/overmind/namespaces/git/state.ts +++ b/packages/app/src/app/overmind/namespaces/git/state.ts @@ -23,7 +23,14 @@ type State = { sourceGitChanges: { [path: string]: GitFileCompare; }; - sourceModulesByPath: { [path: string]: string }; + sourceModulesByPath: { + [path: string]: { + code: string; + isBinary: boolean; + uploadId?: string; + sha?: string; + }; + }; permission: 'admin' | 'write' | 'read'; conflictsResolving: string[]; outOfSyncUpdates: { diff --git a/packages/common/src/types/index.ts b/packages/common/src/types/index.ts index 57cd7b788ce..5d961b6fd1c 100644 --- a/packages/common/src/types/index.ts +++ b/packages/common/src/types/index.ts @@ -65,6 +65,8 @@ export type Module = { insertedAt: string; updatedAt: string; path: string; + uploadId?: string; + sha?: string; type: 'file'; }; @@ -301,6 +303,7 @@ export type GitFileCompare = { deletions: number; filename: string; status: 'added' | 'modified' | 'removed'; + isBinary: boolean; content?: string; }; @@ -718,12 +721,12 @@ export type GitPathChanges = { }; export type GitChanges = { - added: Array<{ path: string; content: string; encoding: 'utf-8' | 'binary' }>; + added: Array<{ path: string; content: string; encoding: 'utf-8' | 'base64' }>; deleted: string[]; modified: Array<{ path: string; content: string; - encoding: 'utf-8' | 'binary'; + encoding: 'utf-8' | 'base64'; }>; }; diff --git a/yarn.lock b/yarn.lock index 767fee25047..a6d86017136 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9001,7 +9001,12 @@ codesandbox-import-util-types@^2.1.9: resolved "https://registry.yarnpkg.com/codesandbox-import-util-types/-/codesandbox-import-util-types-2.1.9.tgz#24ba5ec3d966f51f18b78c48d32e6411da90aa74" integrity sha512-Vc4qh+neVfHtS3RG+7wvaErMoEKdNTnLFnyj4Dcbn3NV7v9nlPj/z6MGhHp9S+vAjegWorFzxg9lKB1WGHTt5Q== -codesandbox-import-utils@^2.1.14, codesandbox-import-utils@^2.1.9: +codesandbox-import-util-types@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/codesandbox-import-util-types/-/codesandbox-import-util-types-2.2.0.tgz#f94799f83838a1e8ba776189a8275870d6579ab6" + integrity sha512-EFXFKCSE7I2VABNa11cgkFJAY8MQCMDhD147xntXw4O/wg5DkGmvor9s0bpk4IYlTOVwZT86H0My1xRhkIzHOA== + +codesandbox-import-utils@^2.1.9: version "2.1.14" resolved "https://registry.yarnpkg.com/codesandbox-import-utils/-/codesandbox-import-utils-2.1.14.tgz#b8222d95208048173ddc754f00618dde5e892eda" integrity sha512-IOV1lk/hEnp6KV4uuHvfjrdIbYSVx11WXr75ABjHPuBh117AgKEbujTduoAChDuEofevV6GwlIyl32EBNECE1Q== @@ -9010,6 +9015,15 @@ codesandbox-import-utils@^2.1.14, codesandbox-import-utils@^2.1.9: istextorbinary "^2.2.1" lz-string "^1.4.4" +codesandbox-import-utils@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/codesandbox-import-utils/-/codesandbox-import-utils-2.2.0.tgz#281b4449746b411cca8934b93fd1817cb4a3e898" + integrity sha512-d2sQnwbsegkjmx6x2Q7RBiLPtSDBKQHSQgekU6h8AimtaNO9d/nc/jWWR/kMnB+T6NorUxCpmoLtHWDqBihFRw== + dependencies: + codesandbox-import-util-types "^2.2.0" + istextorbinary "^2.2.1" + lz-string "^1.4.4" + codesandbox@^2.1.10: version "2.1.10" resolved "https://registry.yarnpkg.com/codesandbox/-/codesandbox-2.1.10.tgz#61cd1ae35b62dd99aae829085335f6270fb514d0"