Run it both in Chrome and FF
- - - diff --git a/scripts/transifex.js b/scripts/transifex.js deleted file mode 100644 index b688156925..0000000000 --- a/scripts/transifex.js +++ /dev/null @@ -1,205 +0,0 @@ -const fs = require('fs').promises; -const spawn = require('cross-spawn'); -const yaml = require('js-yaml'); - -function delay(time) { - return new Promise(resolve => setTimeout(resolve, time)); -} - -function defer() { - const deferred = {}; - deferred.promise = new Promise((resolve, reject) => { - deferred.resolve = resolve; - deferred.reject = reject; - }); - return deferred; -} - -function memoize(fn) { - const cache = {}; - function wrapped(...args) { - const key = args.toString(); - let result = cache[key]; - if (!result) { - result = { data: fn(...args) }; - cache[key] = result; - } - return result.data; - } - return wrapped; -} - -function exec(cmd, args, options) { - return new Promise((resolve, reject) => { - const { stdin, ...rest } = options || {}; - const child = spawn( - cmd, - args, - { ...rest, stdio: ['pipe', 'pipe', 'inherit'] }, - ); - if (stdin != null) { - child.stdin.write(stdin); - child.stdin.end(); - } - const stdoutBuffer = []; - child.stdout.on('data', chunk => { - stdoutBuffer.push(chunk); - }) - child.on('exit', (code) => { - if (code) { - reject(code); - return; - } - const result = Buffer.concat(stdoutBuffer).toString('utf8'); - resolve(result); - }); - }); -} - -let lastRequest; -async function transifexRequest(url, { - method = 'GET', - responseType = 'json', - data = null, -} = {}) { - const deferred = defer(); - const prevRequest = lastRequest; - lastRequest = deferred.promise; - try { - await prevRequest; - let result = await exec( - 'curl', - [ - '-sSL', - '--user', - `api:${process.env.TRANSIFEX_TOKEN}`, - '-X', - method, - '-H', - 'Content-Type: application/json', - ...data == null ? [] : ['-d', '@-'], - `https://www.transifex.com${url}`, - ], - { - stdin: data ? JSON.stringify(data) : null, - }, - ); - if (responseType === 'json') { - result = JSON.parse(result); - } - deferred.resolve(delay(500)); - return result; - } catch (err) { - deferred.reject(err); - throw err; - } -} - -async function getLanguages() { - const result = await transifexRequest('/api/2/project/violentmonkey-nex/?details'); - return result.teams; -} - -async function loadRemote(lang) { - // Reference: https://docs.transifex.com/api/translations#downloading-and-uploading-translations - // Use translated messages since we don't have enough reviewers - const result = await transifexRequest(`/api/2/project/violentmonkey-nex/resource/messagesjson/translation/${lang}/?mode=onlytranslated`); - const remote = JSON.parse(result.content); - return remote; -} - -const loadData = memoize(async function loadData(lang) { - const remote = await loadRemote(lang); - const filePath = `src/_locales/${lang}/messages.yml`; - const local = yaml.safeLoad(await fs.readFile(filePath, 'utf8')); - return { local, remote, filePath }; -}); - -const loadUpdatedLocales = memoize(async function loadUpdatedLocales() { - const diffUrl = process.env.DIFF_URL; - if (!diffUrl) return; - const result = await exec('curl', ['-sSL', diffUrl]); - // Example: - // diff --git a/src/_locales/ko/messages.yml b/src/_locales/ko/messages.yml - const codes = result.split('\n') - .map(line => { - const matches = line.match(/^diff --git a\/src\/_locales\/([^/]+)\/messages.yml b\/src\/_locales\/([^/]+)\/messages.yml$/); - const [, code1, code2] = matches || []; - return code1 === code2 && code1; - }) - .filter(Boolean); - return codes; -}); - -async function pushTranslations(lang) { - const codes = await loadUpdatedLocales(); - // Limit to languages changed in this PR only - if (codes && !codes.includes(lang)) return; - const { local, remote } = await loadData(lang); - const remoteUpdate = {}; - Object.entries(local) - .forEach(([key, value]) => { - const remoteMessage = remote[key] && remote[key].message; - if (value.touched !== false && value.message && value.message !== remoteMessage) remoteUpdate[key] = value; - }); - if (Object.keys(remoteUpdate).length) { - const strings = await transifexRequest(`/api/2/project/violentmonkey-nex/resource/messagesjson/translation/${lang}/strings/`); - const updates = strings.filter(({ key, reviewed }) => !reviewed && remoteUpdate[key]) - .map(({ key, string_hash }) => ({ - source_entity_hash: string_hash, - translation: remoteUpdate[key].message, - })); - process.stdout.write(`\n Uploading translations for ${lang}:\n ${JSON.stringify(updates)}\n`); - await transifexRequest(`/api/2/project/violentmonkey-nex/resource/messagesjson/translation/${lang}/strings/`, { - method: 'PUT', - responseType: 'text', - data: updates, - }); - process.stdout.write(' finished\n'); - } else { - process.stdout.write('up to date\n'); - } -} - -async function pullTranslations(code) { - const { local, remote, filePath } = await loadData(code); - Object.entries(local) - .forEach(([key, value]) => { - const remoteMessage = remote[key] && remote[key].message; - if (remoteMessage) value.message = remoteMessage; - }); - await fs.writeFile(filePath, yaml.safeDump(local), 'utf8'); -} - -async function main() { - let handle; - if (process.argv.includes('push')) handle = pushTranslations; - else if (process.argv.includes('pull')) handle = pullTranslations; - else process.exit(2); - process.stdout.write('Loading languages...'); - const codes = await getLanguages(); - process.stdout.write('OK\n'); - process.stdout.write(`Got ${codes.length} language codes\n`); - for (const code of codes) { - await fs.mkdir(`src/_locales/${code}`, { recursive: true }); - } - spawn.sync('yarn', ['i18n'], { stdio: 'inherit' }); - let current = 0; - const showProgress = (lang) => { - process.stdout.write(`\rLoading translations ${lang} (${current}/${codes.length})...`); - }; - for (const code of codes) { - current += 1; - showProgress(code); - try { - await handle(code); - } catch (err) { - process.stderr.write(`\nError pulling ${code}\n`) - throw err; - } - } - showProgress('OK'); - process.stdout.write('\n'); -} - -main(); diff --git a/scripts/transifex.mjs b/scripts/transifex.mjs new file mode 100644 index 0000000000..4c20362808 --- /dev/null +++ b/scripts/transifex.mjs @@ -0,0 +1,329 @@ +import { readFile, writeFile, mkdir } from 'fs/promises'; +import spawn from 'cross-spawn'; +import yaml from 'js-yaml'; + +const PROJECT_ID = 'o:violentmonkey:p:violentmonkey-nex'; +const RESOURCE_ID = `${PROJECT_ID}:r:messagesjson`; +const RESOURCE_FILE = 'src/_locales/en/messages.yml'; +const request = limitConcurrency(doRequest, 5); + +function delay(time) { + return new Promise(resolve => setTimeout(resolve, time)); +} + +function defer() { + const deferred = {}; + deferred.promise = new Promise((resolve, reject) => { + deferred.resolve = resolve; + deferred.reject = reject; + }); + return deferred; +} + +function memoize(fn) { + const cache = {}; + function wrapped(...args) { + const key = args.toString(); + let result = cache[key]; + if (!result) { + result = { data: fn(...args) }; + cache[key] = result; + } + return result.data; + } + return wrapped; +} + +function limitConcurrency(fn, concurrency) { + const tokens = []; + const processing = new Set(); + async function getToken() { + const token = defer(); + tokens.push(token); + check(); + await token.promise; + return token; + } + function releaseToken(token) { + processing.delete(token); + check(); + } + function check() { + while (tokens.length && processing.size < concurrency) { + const token = tokens.shift(); + processing.add(token); + token.resolve(); + } + } + async function limited(...args) { + const token = await getToken(); + try { + return await fn(...args); + } finally { + releaseToken(token); + } + } + return limited; +} + +async function doRequest(path, options) { + options = { + method: 'GET', + ...options, + }; + const init = { + method: options.method, + headers: { + accept: 'application/vnd.api+json', + ...options.headers, + authorization: `Bearer ${process.env.TRANSIFEX_TOKEN}`, + }, + }; + const qs = options.query ? `?${new URLSearchParams(options.query)}` : ''; + if (options.body) { + init.headers['content-type'] ||= 'application/vnd.api+json'; + init.body = JSON.stringify(options.body); + } + if (!path.includes('://')) path = `https://rest.api.transifex.com${path}`; + const resp = await fetch(path + qs, init); + const isJson = /[+/]json$/.test(resp.headers.get('content-type').split(';')[0]); + const result = await resp[isJson ? 'json' : 'text'](); + if (!resp.ok) throw { resp, result }; + return { resp, result }; +} + +async function uploadResource() { + const source = yaml.load(await readFile(RESOURCE_FILE, 'utf8')); + const content = Object.entries(source).reduce((prev, [key, value]) => { + if (value.touched !== false) { + prev[key] = { + description: value.description, + message: value.message, + }; + } + return prev; + }, {}); + let { result } = await request('/resource_strings_async_uploads', { + method: 'POST', + body: { + data: { + type: 'resource_strings_async_uploads', + attributes: { + content: JSON.stringify(content), + content_encoding: 'text', + }, + relationships: { + resource: { + data: { + id: RESOURCE_ID, + type: 'resources', + }, + }, + }, + }, + }, + }); + while (['pending', 'processing'].includes(result.data.attributes.status)) { + await delay(500); + ({ result } = await request(`/resource_strings_async_uploads/${result.data.id}`)); + } + if (result.data.attributes.status !== 'succeeded') throw { result }; + return result.data.attributes.details; +} + +async function getLanguages() { + const { result } = await request(`/projects/${PROJECT_ID}/languages`); + return result.data.map(({ attributes: { code } }) => code); +} + +async function loadRemote(lang) { + let { resp, result } = await request('/resource_translations_async_downloads', { + method: 'POST', + body: { + data: { + type: 'resource_translations_async_downloads', + attributes: { + mode: 'onlytranslated', + }, + relationships: { + language: { + data: { + id: `l:${lang}`, + type: 'languages', + }, + }, + resource: { + data: { + id: RESOURCE_ID, + type: 'resources', + }, + }, + }, + }, + }, + }); + while (!resp.redirected && ['pending', 'processing', 'succeeded'].includes(result.data.attributes.status)) { + await delay(500); + ({ resp, result } = await request(`/resource_translations_async_downloads/${result.data.id}`)); + } + if (!resp.redirected) throw { resp, result }; + return result; +} + +async function getTranslations(lang) { + let { result } = await request('/resource_translations', { + query: { + 'filter[resource]': RESOURCE_ID, + 'filter[language]': `l:${lang}`, + include: 'resource_string', + }, + }); + let { data, included } = result; + while (result.links.next) { + ({ result } = await request(result.links.next)); + data = data.concat(result.data); + included = included.concat(result.included); + } + const includedMap = included.reduce((prev, item) => { + prev[item.id] = item; + return prev; + }, {}); + data.forEach(item => { + Object.assign(item.relationships.resource_string.data, includedMap[item.relationships.resource_string.data.id]); + }); + return data; +} + +async function updateTranslations(updates) { + const { result } = await request('/resource_translations', { + method: 'PATCH', + headers: { + 'content-type': 'application/vnd.api+json;profile="bulk"', + }, + body: { + data: updates, + }, + }); + return result.data; +} + +const loadData = memoize(async function loadData(lang) { + const remote = await loadRemote(lang); + const filePath = `src/_locales/${lang}/messages.yml`; + const local = yaml.load(await readFile(filePath, 'utf8')); + return { local, remote, filePath }; +}); + +async function loadUpdatedLocales() { + const diffUrl = process.env.DIFF_URL; + if (!diffUrl) return; + const res = await fetch(diffUrl); + const result = await res.text(); + // Example: + // diff --git a/src/_locales/ko/messages.yml b/src/_locales/ko/messages.yml + const langs = result.split('\n') + .map(line => { + const matches = line.match(/^diff --git a\/src\/_locales\/([^/]+)\/messages.yml b\/src\/_locales\/([^/]+)\/messages.yml$/); + const [, code1, code2] = matches || []; + return code1 === code2 && code1; + }) + .filter(Boolean); + return langs; +} + +async function pushTranslations(lang) { + const { local, remote } = await loadData(lang); + const remoteUpdate = {}; + Object.entries(local) + .forEach(([key, value]) => { + const remoteMessage = remote[key] && remote[key].message; + if (value.touched !== false && value.message && value.message !== remoteMessage) remoteUpdate[key] = value; + }); + if (Object.keys(remoteUpdate).length) { + const strings = await getTranslations(lang); + const updates = strings.filter(item => !item.attributes.reviewed && remoteUpdate[item.relationships.resource_string.data.attributes.key]) + .map(item => ({ + id: item.id, + type: item.type, + attributes: { + strings: { + other: remoteUpdate[item.relationships.resource_string.data.attributes.key].message, + }, + }, + })); + process.stdout.write(`\n Uploading translations for ${lang}:\n ${JSON.stringify(updates)}\n`); + await updateTranslations(updates); + process.stdout.write(' finished\n'); + } else { + process.stdout.write('up to date\n'); + } +} + +async function pullTranslations(lang) { + const { local, remote, filePath } = await loadData(lang); + Object.entries(local) + .forEach(([key, value]) => { + const remoteMessage = remote[key] && remote[key].message; + if (remoteMessage) value.message = remoteMessage; + }); + await writeFile(filePath, yaml.dump(local), 'utf8'); +} + +async function batchHandle(handle, allowedLangs) { + process.stdout.write('Loading languages...'); + let langs = await getLanguages(); + process.stdout.write('OK\n'); + process.stdout.write(`Got ${langs.length} language codes\n`); + for (const lang of langs) { + await mkdir(`src/_locales/${lang}`, { recursive: true }); + } + spawn.sync('yarn', ['i18n'], { stdio: 'inherit' }); + if (allowedLangs) langs = langs.filter(lang => allowedLangs.includes(lang)); + let finished = 0; + const showProgress = () => { + process.stdout.write(`\rHandling translations (${finished}/${langs.length})...`); + }; + showProgress(); + await Promise.all(langs.map(async lang => { + try { + await handle(lang); + finished += 1; + showProgress(); + } catch (err) { + process.stderr.write(`\nError handling ${lang}\n`); + throw err; + } + })); + process.stdout.write('\n'); +} + +async function updateResource() { + const result = await uploadResource(); + console.log(Object.entries(result).map(([key, value]) => `${key}: ${value}`).join(', ')); +} + +async function main() { + const [,, command] = process.argv; + switch (command) { + case 'update': { + return updateResource(); + } + case 'push': { + // Limit to languages changed in this PR only + const allowedLangs = await loadUpdatedLocales(); + return batchHandle(pushTranslations, allowedLangs); + } + case 'pull': { + return batchHandle(pullTranslations); + } + default: { + throw new Error(`Unknown command: ${command}`); + } + } +} + +main().catch(err => { + console.error(err); + if (err?.result) console.error('Response:', JSON.stringify(err.result, null, 2)); + process.exitCode = 1; +}); diff --git a/scripts/version-helper.js b/scripts/version-helper.js index 6bd251ad1e..ca60608145 100644 --- a/scripts/version-helper.js +++ b/scripts/version-helper.js @@ -6,12 +6,7 @@ const pkg = require('../package.json'); * > manifest.version = `${pkg.version}.${pkg.beta}` */ function getVersion() { - let version = pkg.version.replace(/-[^.]*/, ''); - if (pkg.beta) version += `.${pkg.beta}`; - // Create a beta release with the same code as in stable release. - // Used in unlisted version. - else if (process.env.BETA) version += 'b'; - return version; + return `${pkg.version.match(/\d+\.\d+/)[0]}.${pkg.beta || 0}`; } function isBeta() { diff --git a/scripts/webpack-base.js b/scripts/webpack-base.js new file mode 100644 index 0000000000..c9b0fc986f --- /dev/null +++ b/scripts/webpack-base.js @@ -0,0 +1,241 @@ +const { resolve } = require('path'); +const { VueLoaderPlugin } = require('vue-loader'); +const HtmlWebpackPlugin = require('html-webpack-plugin'); +const MiniCssExtractPlugin = require('mini-css-extract-plugin'); +const CssMinimizerPlugin = require('css-minimizer-webpack-plugin'); +const TerserPlugin = require('terser-webpack-plugin'); +const deepmerge = require('deepmerge'); +const GroupAssetsPlugin = require('./webpack-group-assets-plugin'); +const { alias, extensions, isProd } = require('./common'); + +const defaultHtmlOptions = { + minify: isProd && { + collapseWhitespace: true, + removeAttributeQuotes: true, + removeComments: true, + removeOptionalTags: true, + removeRedundantAttributes: true, + removeScriptTypeAttributes: true, + removeStyleLinkTypeAttributes: true, + }, + meta: { viewport: 'width=device-width,initial-scale=1.0,minimum-scale=1.0,maximum-scale=1.0' }, +}; +const MIN_OPTS = { + extractComments: false, + parallel: true, + terserOptions: { + compress: { + // `terser` often inlines big one-time functions inside a small "hot" function + reduce_funcs: false, + }, + output: { + ascii_only: true, + comments: false, + wrap_func_args: false, // disabling a premature optimization designed for old browsers + }, + }, +}; +const MIN_OPTS_PUBLIC = isProd && { + include: 'public/', + ...MIN_OPTS, +}; +const MIN_OPTS_MAIN = isProd && deepmerge.all([{}, MIN_OPTS, { + exclude: 'public/', + terserOptions: { + compress: { + ecma: 8, // ES2017 Object.entries and so on + passes: 2, // necessary now since we removed plaid's minimizer + unsafe_arrows: true, // it's 'safe' since we don't rely on function prototypes + }, + }, +}]); +const nodeModules = resolve('node_modules'); + +const pages = [ + 'background', + 'confirm', + 'options', + 'popup', +]; +const createHtmlPage = key => new HtmlWebpackPlugin({ + ...defaultHtmlOptions, + filename: `${key}/index.html`, + chunks: [`${key}/index`], + title: 'Violentmonkey', + scriptLoading: 'blocking', // we don't need `defer` and it breaks in some browsers, see #1632 + // For GroupAssetsPlugin, inject only `index.js` into `body` to avoid FOUC + injectTo: item => ((item.attributes.src || '').endsWith('/index.js') ? 'body' : 'head'), +}); + +const splitVendor = prefix => ({ + [prefix]: { + test: new RegExp(`node_modules[/\\\\]${prefix}`), + name: `public/lib/${prefix}`, + chunks: 'all', + priority: 100, + }, +}); + +function styleLoader(options) { + const { + extract, + loaders = [], + fallback = 'style-loader', + modules = false, + } = options || {}; + const cssLoader = { + loader: 'css-loader', + options: { + modules, + importLoaders: 1, + sourceMap: false, + }, + }; + return [ + extract ? MiniCssExtractPlugin.loader : fallback, + cssLoader, + ...loaders, + ]; +} + +function styleRule(options, rule) { + return { + test: /\.css$/, + use: styleLoader(options), + ...rule, + }; +} + +const styleOptions = { + extract: isProd, +}; +const postcssLoader = { + loader: 'postcss-loader', +}; + +const getBaseConfig = () => ({ + mode: isProd ? 'production' : 'development', + target: 'web', // required by live reloading + devtool: isProd ? false : 'inline-source-map', + output: { + path: resolve('dist'), + publicPath: '/', + filename: '[name].js', + hashFunction: 'xxhash64', + }, + node: { + global: false, + }, + performance: { + maxEntrypointSize: 1e6, + maxAssetSize: 0.5e6, + }, + resolve: { + alias, + extensions, + }, + module: { + rules: [ + // JS/TS + { + test: /\.m?[jt]sx?$/, + use: 'babel-loader', + exclude: file => /node_modules/.test(file) && !/vueleton|@vue[/\\]shared/.test(file), + }, + // CSS + { + oneOf: [ + // library CSS files: node_modules/**/*.css + styleRule(styleOptions, { + include: [nodeModules], + }), + // CSS modules: src/**/*.module.css + styleRule({ + ...styleOptions, + loaders: [postcssLoader], + modules: {}, + }, { + test: /\.module\.css$/, + }), + // normal CSS files: src/**/*.css + styleRule({ + ...styleOptions, + loaders: [postcssLoader], + }), + ], + }, + // SVG + { + test: /\.svg$/, + use: [{ + loader: 'svg-sprite-loader', + options: { + // extract: extractSVG, + }, + }], + include: [resolve('src/resources/svg')], + }, + // Vue + { + test: /\.vue$/, + loader: 'vue-loader', + options: { + babelParserPlugins: ['functionBind'], + compilerOptions: { + whitespace: 'condense', + }, + }, + }, + ], + }, + optimization: { + runtimeChunk: false, + splitChunks: { + cacheGroups: { + 'common-ui': { + name: 'common-ui', + test: new RegExp([ + /\bsvg/, + // don't extract CSS as it'll change the relative order of rules which breaks appearance + 'src/common/(?!zip|.*\\.css$)', + 'node_modules/@violentmonkey/shortcut', + 'node_modules/@?vue', + ].map(re => re.source || re).join('|').replace(/\\?\//g, '[/\\\\]')), + chunks: c => ![ + 'background/index', // only 4kB of common code + 'injected', + 'injected-web', + ].includes(c.name), + }, + ...splitVendor('codemirror'), + }, + }, + minimizer: isProd ? [ + new CssMinimizerPlugin(), + new TerserPlugin(MIN_OPTS_PUBLIC), + new TerserPlugin(MIN_OPTS_MAIN), + ] : [], + }, + plugins: [ + new VueLoaderPlugin(), + new GroupAssetsPlugin(), + ...styleOptions.extract ? [new MiniCssExtractPlugin({ + filename: '[name].css', + })] : [], + require('unplugin-icons/webpack')(), + ], +}); + +const getPageConfig = () => { + const config = getBaseConfig(); + config.entry = Object.fromEntries(pages.map(name => [`${name}/index`, `./src/${name}`])); + config.plugins = [ + ...config.plugins, + ...pages.filter(key => key !== 'background').map(createHtmlPage), + ]; + return config; +}; + +exports.isProd = isProd; +exports.getBaseConfig = getBaseConfig; +exports.getPageConfig = getPageConfig; diff --git a/scripts/webpack-group-assets-plugin.js b/scripts/webpack-group-assets-plugin.js new file mode 100644 index 0000000000..0ffbed4d55 --- /dev/null +++ b/scripts/webpack-group-assets-plugin.js @@ -0,0 +1,36 @@ +const HtmlWebpackPlugin = require('html-webpack-plugin'); + +class GroupAssetsPlugin { + apply(compiler) { + compiler.hooks.compilation.tap('GroupAssetsPlugin', (compilation) => { + HtmlWebpackPlugin.getHooks(compilation).alterAssetTagGroups.tapAsync( + 'GroupAssetsPlugin', (data, callback) => { + this.groupAssets(data); + // console.log('grouped', data); + callback(null, data); + } + ); + }); + } + + groupAssets(data) { + const { injectTo } = data.plugin.options; + if (typeof injectTo === 'function') { + const groups = { head: [], body: [] }; + [ + ['head', data.headTags], + ['body', data.bodyTags], + ].forEach(([defaultGroup, items]) => { + items.forEach(item => { + const groupName = injectTo(item, defaultGroup); + const group = groups[groupName] || groups[defaultGroup]; + group.push(item); + }); + }); + data.headTags = groups.head; + data.bodyTags = groups.body; + } + } +} + +module.exports = GroupAssetsPlugin; diff --git a/scripts/webpack-protect-bootstrap-plugin.js b/scripts/webpack-protect-bootstrap-plugin.js new file mode 100644 index 0000000000..751ff962d4 --- /dev/null +++ b/scripts/webpack-protect-bootstrap-plugin.js @@ -0,0 +1,109 @@ +const escapeStringRegexp = require('escape-string-regexp'); +const webpack = require('webpack'); + +const G = webpack.RuntimeGlobals; +const OBJ_RULE = [ + /([[(,=:]\s*{)(?!__proto__:)\s*(.)/g, + (_, str, next) => `${str}__proto__: null${next === '}' ? '' : ','}${next}`, +]; +const BOOTSTRAP_RULES = [ + OBJ_RULE, + [ + "typeof Symbol !== 'undefined' && Symbol.toStringTag", + 'true', + G.makeNamespaceObject, + ], [ + 'Symbol.toStringTag', + 'toStringTagSym', + G.makeNamespaceObject, + ], [ + 'Object.defineProperty(', + 'defineProperty(', + G.definePropertyGetters, + ], [ + `${G.hasOwnProperty}(definition, key) && !${G.hasOwnProperty}(exports, key)`, + '!(key in exports)', + G.definePropertyGetters, + ], [ + 'Object.prototype.hasOwnProperty.call(', + 'hasOwnProperty(', + G.hasOwnProperty, + ], +]; +const MAIN_RULES = [ + [ + /(__webpack_modules__\[moduleId])\.call\(/g, + 'safeCall($1, ', + false, + ], [ + new RegExp(`(?:${[ + '// webpackBootstrap\\s+(?:/\\*+/\\s*)?"use strict";\\s+', + `var (__webpack_module_cache__|${G.require}) = {};.*?`, + ].join('|')})var ${G.exports} =`, 's'), + patchBootstrap, + false, + ], [ + new RegExp(`(${[ + `${G.definePropertyGetters}\\(${G.exports}, {`, + `var ${G.exports} = {`, + `var __webpack_modules__ = \\({`, + ].join('|')})(?!__proto__:)\\s*(.)`, 'g'), + OBJ_RULE[1], + false, + ], +]; + +/** + * WARNING! The following globals must be correctly assigned using wrapper-webpack-plugin. + * toStringTagSym = Symbol.toStringTag + * defineProperty = Object.defineProperty + * hasOwnProperty = Object.prototype.hasOwnProperty + * safeCall = Function.prototype.call.bind(Function.prototype.call) + */ +class WebpackProtectBootstrapPlugin { + apply(compiler) { + const NAME = WebpackProtectBootstrapPlugin.name; + compiler.hooks.compilation.tap(NAME, (compilation) => { + const hooks = webpack.javascript.JavascriptModulesPlugin.getCompilationHooks(compilation); + hooks.renderMain.tap(NAME, replace.bind(null, MAIN_RULES)); + }); + } +} + +function patchBootstrap(src, moduleCache) { + if (!moduleCache) return ' ' + src; // everything was concatenated + const props = src.match(new RegExp(`(?<=\\b${G.require}\\.)(\\w+)(?= = )`, 'g')); + const guard = props + ? `for (let i = 0, props="${[...new Set(props)].join('')}"; i < props.length; i++) + defineProperty(${G.require}, props[i], {__proto__: null, value: 0, writable: 1});\n` + : ''; + const res = replace(BOOTSTRAP_RULES, src, this); + // splicing the guard after the first line to handle `require = {}` + const i = res.indexOf('\n'); + return res.slice(0, i) + guard + res.slice(i); +} + +function replace(rules, src, info) { + src = src.source?.() || src; + let res = src; + for (const rule of rules) { + const [from, to, test = true] = rule; + const fromRe = typeof from === 'string' + ? new RegExp(escapeStringRegexp(from), 'g') + : from; + const dst = res.replace(fromRe, to.bind?.(info) || to); + const mandatory = test === true + || test.test?.(src) + || typeof test === 'string' && src.includes(test); + if (dst === res && mandatory) { + const err = `[${WebpackProtectBootstrapPlugin.name}] ` + + `"${from}" not found in ${info.chunk.name || 'bootstrap'}`; + console.warn(`${err}:\n${src.slice(0, 500)}`); // this prints immediately + throw new Error(err); // this prints at the end of build + } + res = dst; + } + return res; +} + +module.exports = WebpackProtectBootstrapPlugin; diff --git a/scripts/webpack-util.js b/scripts/webpack-util.js new file mode 100644 index 0000000000..2404eaf8ce --- /dev/null +++ b/scripts/webpack-util.js @@ -0,0 +1,101 @@ +const fs = require('fs'); +const babelCore = require('@babel/core'); +const webpack = require('webpack'); + +const entryGlobals = { + 'common': [], + 'injected/content': [], + 'injected/web': [], +}; +const entryPathToFilename = path => path === '*' + ? `./src/common/safe-globals-shared.js` + : `./src/${path}/safe-globals.js`; +Object.entries(entryGlobals).forEach(([name, val]) => { + const parts = name.split('/'); + if (parts[1]) parts[1] = name; + val.push('*', ...parts); +}); + +exports.restrictedSyntax = ( + // Hiding `code` so eslint doesn't complain about invalid schema + rules => rules.map(r => ( + Object.defineProperty(r, 'code', { enumerable: false, value: r.code }) + )) +)([{ + selector: 'ArrayPattern', + message: 'Destructuring via Symbol.iterator may be spoofed/broken in an unsafe environment', + code: '[window.foo]=[]', +}, { + selector: ':matches(ArrayExpression, CallExpression) > SpreadElement', + message: 'Spreading via Symbol.iterator may be spoofed/broken in an unsafe environment', + code: 'open([...[]])', +}, { + selector: '[callee.object.name="Object"], MemberExpression[object.name="Object"]', + message: 'Using potentially spoofed methods in an unsafe environment', + code: 'Object.assign()', + // TODO: auto-generate the rule using GLOBALS +}, { + selector: `CallExpression[callee.name="defineProperty"]:not(${[ + '[arguments.2.properties.0.key.name="__proto__"]', + ':has(CallExpression[callee.name="nullObjFrom"])' + ].join(',')})`, + message: 'Prototype of descriptor may be spoofed/broken in an unsafe environment', + code: 'defineProperty(open, "foo", {foo:1})', +}]); + +/** + * Adds a watcher for files in entryGlobals to properly recompile the project on changes. + */ +function addWrapperWithGlobals(name, config, defsObj, callback) { + config.module.rules.push({ + test: new RegExp(`/${name}/.*?\\.js$`.replace(/\//g, /[/\\]/.source)), + use: [{ + loader: './scripts/fake-dep-loader.js', + options: { files: entryGlobals[name].map(entryPathToFilename) }, + }], + }); + const defsRe = new RegExp(`\\b(${ + Object.keys(defsObj) + .join('|') + .replace(/\./g, '\\.') + })\\b`, 'g'); + const reader = () => ( + entryGlobals[name] + .map(path => readGlobalsFile(path)) + .join('\n') + .replace(defsRe, s => defsObj[s]) + ); + const { header, footer, test } = callback(reader); + config.plugins.push( + new webpack.BannerPlugin({ test, raw: true, banner: header }), + new webpack.BannerPlugin({ test, raw: true, banner: footer, footer: true }) + ); +} + +function getCodeMirrorThemes() { + const name = 'neo.css'; + return fs.readdirSync( + require.resolve(`codemirror/theme/${name}`).slice(0, -name.length), + { withFileTypes: true }, + ).map(e => e.isFile() && e.name.endsWith('.css') && e.name.slice(0, -4)) + .filter(Boolean); +} + +function readGlobalsFile(path, babelOpts = {}) { + const { ast, code = !ast } = babelOpts; + const filename = entryPathToFilename(path); + const src = fs.readFileSync(filename, { encoding: 'utf8' }) + .replace(/\bexport\s+(function\s+(\w+))/g, 'const $2 = $1') + .replace(/\bexport\s+(?=(const|let)\s)/g, ''); + const res = babelCore.transformSync(src, { + ...babelOpts, + ast, + code, + filename, + }); + return ast ? res : res.code; +} + +exports.addWrapperWithGlobals = addWrapperWithGlobals; +exports.getCodeMirrorThemes = getCodeMirrorThemes; +exports.readGlobalsFile = readGlobalsFile; diff --git a/scripts/webpack.conf.js b/scripts/webpack.conf.js index 44ccbf4b6a..b545258d2e 100644 --- a/scripts/webpack.conf.js +++ b/scripts/webpack.conf.js @@ -1,110 +1,110 @@ -const { modifyWebpackConfig, shallowMerge, defaultOptions } = require('@gera2ld/plaid'); -const { isProd } = require('@gera2ld/plaid/util'); const webpack = require('webpack'); -const WrapperWebpackPlugin = require('wrapper-webpack-plugin'); -const TerserPlugin = require('terser-webpack-plugin'); -const projectConfig = require('./plaid.conf'); -const mergedConfig = shallowMerge(defaultOptions, projectConfig); +const { ListBackgroundScriptsPlugin } = require('./manifest-helper'); +const { addWrapperWithGlobals, getCodeMirrorThemes } = require('./webpack-util'); +const ProtectWebpackBootstrapPlugin = require('./webpack-protect-bootstrap-plugin'); +const { getVersion } = require('./version-helper'); +const { configLoader } = require('./config-helper'); +const { getBaseConfig, getPageConfig, isProd } = require('./webpack-base'); -const INIT_FUNC_NAME = 'VMInitInjection'; -// Copied from gulpfile.js: strip alphabetic suffix -const VM_VER = require('../package.json').version.replace(/-[^.]*/, ''); +// Avoiding collisions with globals of a content-mode userscript +const INIT_FUNC_NAME = '**VMInitInjection**'; +const VAULT_ID = 'VAULT_ID'; +const PAGE_MODE_HANDSHAKE = 'PAGE_MODE_HANDSHAKE'; +const VM_VER = getVersion(); -const definitions = new webpack.DefinePlugin({ - 'process.env.INIT_FUNC_NAME': JSON.stringify(INIT_FUNC_NAME), - 'process.env.DEBUG': JSON.stringify(process.env.DEBUG || false), - 'process.env.VM_VER': JSON.stringify(VM_VER), -}); -const minimizerOptions = { - cache: true, - parallel: true, - sourceMap: true, - terserOptions: { - output: { - ascii_only: true, - }, - }, -}; -const minimizer = isProd && [ - new TerserPlugin({ - chunkFilter: ({ name }) => !name.startsWith('public/'), - ...minimizerOptions, - terserOptions: { - ...minimizerOptions.terserOptions, - compress: { - ecma: 6, - // 'safe' since we don't rely on function prototypes - unsafe_arrows: true, - }, - }, - }), - new TerserPlugin({ - chunkFilter: ({ name }) => name.startsWith('public/'), - ...minimizerOptions, - }), -]; +global.localStorage = {}; // workaround for node 25 and HtmlWebpackPlugin's `...global` + +configLoader + // Default values + .add({ + DEBUG: false, + }) + // Load from `./.env` + .envFile() + // Load from `process.env` + .env() + // Override values + .add({ + VM_VER, + }); -const modify = (extra, init) => modifyWebpackConfig( - (config) => { - config.plugins.push(definitions); - if (init) init(config); - return config; - }, { - projectConfig: { - ...mergedConfig, - ...extra, - optimization: { - ...mergedConfig.optimization, - ...(extra || {}).pages && { - runtimeChunk: false, - splitChunks: false, - }, - minimizer, - }, - }, - }, -); +const pickEnvs = (items) => { + return Object.assign({}, ...items.map(key => ({ + [`process.env.${key}`]: JSON.stringify(configLoader.get(key)), + }))); +}; +const defsObj = { + ...pickEnvs([ + 'DEBUG', + 'VM_VER', + 'SYNC_GOOGLE_DESKTOP_ID', + 'SYNC_GOOGLE_DESKTOP_SECRET', + 'SYNC_ONEDRIVE_CLIENT_ID', + 'SYNC_DROPBOX_CLIENT_ID', + ]), + 'process.env.INIT_FUNC_NAME': JSON.stringify(INIT_FUNC_NAME), + 'process.env.CODEMIRROR_THEMES': JSON.stringify(getCodeMirrorThemes()), + 'process.env.DEV': JSON.stringify(!isProd), + 'process.env.TEST': JSON.stringify(process.env.BABEL_ENV === 'test'), +}; // avoid running webpack bootstrap in a potentially hacked environment // after documentElement was replaced which triggered reinjection of content scripts -const skipReinjectionHeader = `if (window['${INIT_FUNC_NAME}'] !== 1)`; -const skipReinjectionConfig = (config, test) => config.plugins.push( - new WrapperWebpackPlugin({ - header: skipReinjectionHeader, - ...test && { test }, +const skipReinjectionHeader = `{ + const INIT_FUNC_NAME = '${INIT_FUNC_NAME}'; + if (window[INIT_FUNC_NAME] !== 1)`; + +const buildConfig = (page, entry, init) => { + const config = entry ? getBaseConfig() : getPageConfig(); + config.plugins.push(new webpack.DefinePlugin({ + ...defsObj, + // Conditional compilation to remove unsafe and unused stuff from `injected` + 'process.env.IS_INJECTED': JSON.stringify(/injected/.test(page) && page), })); + if (typeof entry === 'string') { + config.entry = { [page]: entry }; + } + if (!entry) init = page; + if (init) init(config); + return config; +}; -module.exports = Promise.all([ - modify(null, config => { - config.output.publicPath = '/'; - skipReinjectionConfig(config, /^browser\.js$/); +module.exports = [ + buildConfig((config) => { + addWrapperWithGlobals('common', config, defsObj, getGlobals => ({ + header: () => `{ ${getGlobals()}`, + footer: '}', + test: /^(?!injected|public).*\.js$/, + })); + config.plugins.push(new ListBackgroundScriptsPlugin({ + minify: false, // keeping readable + })); + (config.ignoreWarnings ??= []).push({ + // suppressing a false warning (the HTML spec allows it) as we don't need SSR + message: /